Skip to content

Commit bee3f8d

Browse files
authored
feat(format): introduce avro file format (alibaba#84)
1 parent 124d6ff commit bee3f8d

16 files changed

Lines changed: 1813 additions & 0 deletions
Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#include "paimon/format/avro/avro_file_format.h"
20+
21+
#include <memory>
22+
#include <utility>
23+
24+
#include "arrow/c/bridge.h"
25+
#include "arrow/c/helpers.h"
26+
#include "paimon/common/utils/arrow/status_utils.h"
27+
#include "paimon/format/avro/avro_reader_builder.h"
28+
#include "paimon/format/avro/avro_stats_extractor.h"
29+
#include "paimon/format/avro/avro_writer_builder.h"
30+
#include "paimon/status.h"
31+
32+
namespace arrow {
33+
class Schema;
34+
} // namespace arrow
35+
struct ArrowSchema;
36+
37+
namespace paimon::avro {
38+
39+
AvroFileFormat::AvroFileFormat(const std::map<std::string, std::string>& options)
40+
: identifier_("avro"), options_(options) {}
41+
42+
Result<std::unique_ptr<ReaderBuilder>> AvroFileFormat::CreateReaderBuilder(
43+
int32_t batch_size) const {
44+
return std::make_unique<AvroReaderBuilder>(options_, batch_size);
45+
}
46+
47+
Result<std::unique_ptr<WriterBuilder>> AvroFileFormat::CreateWriterBuilder(
48+
::ArrowSchema* schema, int32_t batch_size) const {
49+
if (schema == nullptr) {
50+
return Status::Invalid("schema is nullptr");
51+
}
52+
PAIMON_ASSIGN_OR_RAISE_FROM_ARROW(std::shared_ptr<arrow::Schema> typed_schema,
53+
arrow::ImportSchema(schema));
54+
return std::make_unique<AvroWriterBuilder>(typed_schema, batch_size, options_);
55+
}
56+
57+
Result<std::unique_ptr<FormatStatsExtractor>> AvroFileFormat::CreateStatsExtractor(
58+
::ArrowSchema* schema) const {
59+
ArrowSchemaRelease(schema);
60+
return std::make_unique<AvroStatsExtractor>(options_);
61+
}
62+
63+
} // namespace paimon::avro
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#pragma once
20+
21+
#include <cstdint>
22+
#include <map>
23+
#include <memory>
24+
#include <string>
25+
26+
#include "paimon/format/file_format.h"
27+
#include "paimon/format/format_stats_extractor.h"
28+
#include "paimon/format/reader_builder.h"
29+
#include "paimon/format/writer_builder.h"
30+
#include "paimon/result.h"
31+
32+
struct ArrowSchema;
33+
34+
namespace paimon::avro {
35+
36+
class AvroFileFormat : public FileFormat {
37+
public:
38+
explicit AvroFileFormat(const std::map<std::string, std::string>& options);
39+
40+
const std::string& Identifier() const override {
41+
return identifier_;
42+
}
43+
Result<std::unique_ptr<ReaderBuilder>> CreateReaderBuilder(int32_t batch_size) const override;
44+
Result<std::unique_ptr<WriterBuilder>> CreateWriterBuilder(::ArrowSchema* schema,
45+
int32_t batch_size) const override;
46+
Result<std::unique_ptr<FormatStatsExtractor>> CreateStatsExtractor(
47+
::ArrowSchema* schema) const override;
48+
49+
private:
50+
const std::string identifier_;
51+
const std::map<std::string, std::string> options_;
52+
};
53+
54+
} // namespace paimon::avro
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#include "paimon/format/avro/avro_file_format_factory.h"
20+
21+
#include <utility>
22+
23+
#include "paimon/factories/factory.h"
24+
#include "paimon/format/avro/avro_file_format.h"
25+
26+
namespace paimon::avro {
27+
28+
const char AvroFileFormatFactory::IDENTIFIER[] = "avro";
29+
30+
Result<std::unique_ptr<FileFormat>> AvroFileFormatFactory::Create(
31+
const std::map<std::string, std::string>& options) const {
32+
return std::make_unique<AvroFileFormat>(options);
33+
}
34+
35+
static __attribute__((constructor)) void AvroFileFormatFactoryRegisterLogicalTypes() {
36+
::avro::CustomLogicalTypeRegistry::instance().registerType(
37+
"map", [](const std::string&) { return std::make_shared<MapLogicalType>(); });
38+
}
39+
40+
REGISTER_PAIMON_FACTORY(AvroFileFormatFactory);
41+
42+
} // namespace paimon::avro
Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
#pragma once
20+
21+
#include <map>
22+
#include <memory>
23+
#include <string>
24+
25+
#include "avro/LogicalType.hh"
26+
#include "paimon/format/file_format.h"
27+
#include "paimon/format/file_format_factory.h"
28+
#include "paimon/result.h"
29+
30+
namespace paimon::avro {
31+
32+
struct MapLogicalType : public ::avro::CustomLogicalType {
33+
MapLogicalType() : ::avro::CustomLogicalType("map") {}
34+
};
35+
36+
class AvroFileFormatFactory : public FileFormatFactory {
37+
public:
38+
static const char IDENTIFIER[];
39+
40+
const char* Identifier() const override {
41+
return IDENTIFIER;
42+
}
43+
44+
Result<std::unique_ptr<FileFormat>> Create(
45+
const std::map<std::string, std::string>& options) const override;
46+
};
47+
48+
} // namespace paimon::avro

0 commit comments

Comments
 (0)