Skip to content

Commit a175073

Browse files
authored
feat: add IO stream infrastructure (alibaba#39)
1 parent 9fd9edb commit a175073

17 files changed

Lines changed: 2091 additions & 0 deletions
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
#include <cstdint>
22+
#include <functional>
23+
#include <memory>
24+
#include <string>
25+
26+
#include "paimon/fs/file_system.h"
27+
#include "paimon/result.h"
28+
#include "paimon/status.h"
29+
#include "paimon/visibility.h"
30+
31+
namespace paimon {
32+
class Bytes;
33+
class MemoryPool;
34+
35+
/// A buffered input stream that wraps another `InputStream` to provide buffering capabilities.
36+
///
37+
/// `BufferedInputStream` improves I/O performance by reducing the number of system calls
38+
/// through internal buffering. It reads data from the underlying stream in larger chunks
39+
/// and serves subsequent read requests from the internal buffer when possible.
40+
class PAIMON_EXPORT BufferedInputStream : public InputStream {
41+
public:
42+
/// Creates a new buffered input stream that wraps the provided input stream.
43+
/// The buffer is allocated from the specified memory pool.
44+
///
45+
/// @param in The underlying input stream to wrap.
46+
/// @param buffer_size Size of the internal buffer in bytes.
47+
/// @param pool Memory pool for buffer allocation.
48+
BufferedInputStream(const std::shared_ptr<InputStream>& in, int32_t buffer_size,
49+
MemoryPool* pool);
50+
51+
~BufferedInputStream() noexcept override;
52+
53+
Status Seek(int64_t offset, SeekOrigin origin) override;
54+
55+
Result<int64_t> GetPos() const override;
56+
57+
Result<int32_t> Read(char* buffer, uint32_t size) override;
58+
59+
Result<int32_t> Read(char* buffer, uint32_t size, uint64_t offset) override;
60+
61+
void ReadAsync(char* buffer, uint32_t size, uint64_t offset,
62+
std::function<void(Status)>&& callback) override;
63+
64+
Result<uint64_t> Length() const override;
65+
66+
Status Close() override;
67+
68+
Result<std::string> GetUri() const override;
69+
70+
static constexpr int32_t DEFAULT_BUFFER_SIZE = 8192;
71+
72+
private:
73+
/// Fill the internal buffer from the underlying stream.
74+
Status Fill();
75+
76+
/// Internal read implementation.
77+
/// @pre size > 0
78+
Result<int32_t> InnerRead(char* buffer, int32_t size);
79+
80+
/// Validate that the expected number of bytes were read.
81+
Status AssertReadLength(int32_t read_length, int32_t actual_read_length) const;
82+
83+
private:
84+
int32_t buffer_size_;
85+
int32_t pos_ = 0;
86+
int32_t count_ = 0;
87+
std::unique_ptr<Bytes> buffer_;
88+
std::shared_ptr<InputStream> in_;
89+
};
90+
91+
} // namespace paimon
Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
#include <cstdint>
23+
#include <functional>
24+
#include <string>
25+
26+
#include "paimon/fs/file_system.h"
27+
#include "paimon/result.h"
28+
#include "paimon/status.h"
29+
#include "paimon/visibility.h"
30+
31+
namespace paimon {
32+
/// Input stream for memory buffer, inherits from `InputStream`.
33+
class PAIMON_EXPORT ByteArrayInputStream : public InputStream {
34+
public:
35+
ByteArrayInputStream(const char* buffer, uint64_t length);
36+
~ByteArrayInputStream() override = default;
37+
38+
/// @return The raw data pointer of current pos.
39+
const char* GetRawData() const;
40+
41+
Status Seek(int64_t offset, SeekOrigin origin) override;
42+
43+
Result<int64_t> GetPos() const override {
44+
return position_;
45+
}
46+
47+
Result<int32_t> Read(char* buffer, uint32_t size) override;
48+
49+
Result<int32_t> Read(char* buffer, uint32_t size, uint64_t offset) override;
50+
51+
void ReadAsync(char* buffer, uint32_t size, uint64_t offset,
52+
std::function<void(Status)>&& callback) override;
53+
54+
Result<uint64_t> Length() const override {
55+
return length_;
56+
}
57+
58+
Status Close() override;
59+
60+
Result<std::string> GetUri() const override;
61+
62+
private:
63+
const char* buffer_;
64+
const uint64_t length_;
65+
int64_t position_;
66+
};
67+
} // namespace paimon
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
#pragma once
21+
22+
#include <cstdint>
23+
#include <memory>
24+
#include <string>
25+
26+
#include "paimon/io/byte_order.h"
27+
#include "paimon/result.h"
28+
#include "paimon/status.h"
29+
#include "paimon/visibility.h"
30+
31+
namespace paimon {
32+
class Bytes;
33+
class InputStream;
34+
35+
/// `DataInputStream` provides a convenient wrapper around `InputStream` for reading typed data.
36+
/// @note The default byte order is big-endian to maintain compatibility with the Java
37+
/// implementation.
38+
class PAIMON_EXPORT DataInputStream {
39+
public:
40+
/// Constructs a `DataInputStream` wrapping the given `InputStream`.
41+
/// @param input_stream The underlying input stream to read from.
42+
explicit DataInputStream(const std::shared_ptr<InputStream>& input_stream);
43+
44+
/// Seek to a specific position in the underlying input stream.
45+
/// @param offset The absolute byte offset to seek to.
46+
Status Seek(int64_t offset) const;
47+
48+
/// Read a typed value from the stream.
49+
/// @return Result containing the read value or an error status.
50+
template <typename T>
51+
Result<T> ReadValue() const;
52+
53+
/// Read some bytes to a `Bytes` object from the stream. The length of bytes is the number of
54+
/// bytes read from the stream.
55+
/// @param bytes Buffer to store the read bytes.
56+
Status ReadBytes(Bytes* bytes) const;
57+
58+
/// Read raw data of specified size from the stream.
59+
/// @param data Buffer to store the read data.
60+
/// @param size Number of bytes to read.
61+
Status Read(char* data, uint32_t size) const;
62+
63+
/// Read string from the stream.
64+
/// @note First read length (int16), then read string bytes.
65+
Result<std::string> ReadString() const;
66+
67+
/// Get the current position in the underlying input stream.
68+
Result<int64_t> GetPos() const;
69+
70+
/// Get the total length of the underlying input stream.
71+
Result<uint64_t> Length() const;
72+
73+
/// Set the byte order for endianness conversion.
74+
/// @param order The byte order to use `PAIMON_BIG_ENDIAN` or `PAIMON_LITTLE_ENDIAN`.
75+
void SetOrder(ByteOrder order) {
76+
byte_order_ = order;
77+
}
78+
79+
private:
80+
/// Validate that the expected number of bytes were read.
81+
/// @param read_length Expected number of bytes to read.
82+
/// @param actual_read_length Actual number of bytes read.
83+
Status AssertReadLength(int32_t read_length, int32_t actual_read_length) const;
84+
85+
/// Check if there are enough bytes available to read.
86+
/// @param need_length Number of bytes needed.
87+
Status AssertBoundary(int32_t need_length) const;
88+
89+
/// Determine if byte swapping is needed based on current byte order and system endianness.
90+
/// @return `true` if byte swapping is required, `false` otherwise.
91+
bool NeedSwap() const;
92+
93+
private:
94+
std::shared_ptr<InputStream> input_stream_;
95+
ByteOrder byte_order_ = ByteOrder::PAIMON_BIG_ENDIAN;
96+
};
97+
} // namespace paimon

0 commit comments

Comments
 (0)