|
43 | 43 | #include "exprs/vexpr_context.h" |
44 | 44 | #include "format/json/new_json_reader.h" |
45 | 45 | #include "format_v2/column_data.h" |
| 46 | +#include "format_v2/table_reader.h" |
46 | 47 | #include "io/io_common.h" |
47 | 48 | #include "runtime/descriptors.h" |
48 | 49 | #include "runtime/runtime_profile.h" |
@@ -856,4 +857,165 @@ TEST(JsonReaderTest, AppliesDeleteAndNormalConjunctsWithPredicateFilterAccountin |
856 | 857 | EXPECT_EQ(io_ctx->predicate_filtered_rows, 1); |
857 | 858 | } |
858 | 859 |
|
| 860 | +// Partition-only scans have no file-local output columns, but still need one logical row per |
| 861 | +// accepted JSON record so TableReader can materialize the partition constants. |
| 862 | +TEST(JsonReaderTest, EmptyFileProjectionPreservesRows) { |
| 863 | + ObjectPool pool; |
| 864 | + const auto slots = build_slots(&pool); |
| 865 | + for (const auto& source_slots : {slots, std::vector<SlotDescriptor*> {}}) { |
| 866 | + auto result = read_once("empty_projection.jsonl", "{\"id\":1}\n\n{}\n{\"id\":3}\n", |
| 867 | + json_scan_params(), source_slots, {}, true, true); |
| 868 | + ASSERT_TRUE(result.status.ok()) << result.status; |
| 869 | + EXPECT_EQ(result.block.columns(), 0); |
| 870 | + EXPECT_EQ(result.rows, 3); |
| 871 | + EXPECT_FALSE(result.eof); |
| 872 | + EXPECT_TRUE(result.second_status.ok()); |
| 873 | + EXPECT_EQ(result.second_rows, 0); |
| 874 | + EXPECT_TRUE(result.second_eof); |
| 875 | + } |
| 876 | +} |
| 877 | + |
| 878 | +TEST(JsonReaderTest, EmptyFileProjectionPreservesOuterArrayRows) { |
| 879 | + auto result = read_once("empty_projection_array.json", R"([{"id":1},{},{"id":3}])", |
| 880 | + json_scan_params(false, true), {}, {}, true, true); |
| 881 | + ASSERT_TRUE(result.status.ok()) << result.status; |
| 882 | + EXPECT_EQ(result.rows, 3); |
| 883 | + EXPECT_TRUE(result.second_status.ok()); |
| 884 | + EXPECT_TRUE(result.second_eof); |
| 885 | +} |
| 886 | + |
| 887 | +TEST(JsonReaderTest, EmptyFileProjectionPreservesJsonRootRows) { |
| 888 | + auto result = read_once("empty_projection_root.jsonl", |
| 889 | + "{\"payload\":{\"id\":1}}\n{\"payload\":{\"id\":2}}\n", |
| 890 | + json_scan_params(true, false, "", "$.payload"), {}, {}, true, true); |
| 891 | + ASSERT_TRUE(result.status.ok()) << result.status; |
| 892 | + EXPECT_EQ(result.rows, 2); |
| 893 | + EXPECT_TRUE(result.second_status.ok()); |
| 894 | + EXPECT_TRUE(result.second_eof); |
| 895 | +} |
| 896 | + |
| 897 | +TEST(JsonReaderTest, EmptyFileProjectionHonorsJsonPaths) { |
| 898 | + ObjectPool pool; |
| 899 | + const auto slots = build_slots(&pool); |
| 900 | + auto result = read_once("empty_projection_jsonpaths.jsonl", |
| 901 | + "{\"payload\":{\"id\":1}}\n{\"other\":2}\n{\"payload\":{\"id\":3}}\n", |
| 902 | + json_scan_params(true, false, R"(["$.payload.id","$.payload.user"])"), |
| 903 | + slots, {}, true, true); |
| 904 | + ASSERT_TRUE(result.status.ok()) << result.status; |
| 905 | + EXPECT_EQ(result.rows, 2); |
| 906 | + EXPECT_TRUE(result.second_status.ok()); |
| 907 | + EXPECT_TRUE(result.second_eof); |
| 908 | +} |
| 909 | + |
| 910 | +TEST(JsonReaderTest, EmptyFileProjectionPreservesRootJsonPathRows) { |
| 911 | + auto result = read_once("empty_projection_root_path.jsonl", "{\"id\":1}\n{}\n", |
| 912 | + json_scan_params(true, false, R"(["$."])"), {}, {}, true, true); |
| 913 | + ASSERT_TRUE(result.status.ok()) << result.status; |
| 914 | + EXPECT_EQ(result.rows, 2); |
| 915 | + EXPECT_TRUE(result.second_eof); |
| 916 | +} |
| 917 | + |
| 918 | +TEST(JsonReaderTest, EmptyFileProjectionPreservesRootedArrayJsonPathRows) { |
| 919 | + auto result = read_once("empty_projection_rooted_array.json", |
| 920 | + R"({"rows":[{"payload":{"id":1}},{"other":2},{"payload":{"id":3}}]})", |
| 921 | + json_scan_params(false, true, R"(["$.payload.id"])", "$.rows"), {}, {}, |
| 922 | + true, true); |
| 923 | + ASSERT_TRUE(result.status.ok()) << result.status; |
| 924 | + EXPECT_EQ(result.rows, 2); |
| 925 | + EXPECT_TRUE(result.second_status.ok()); |
| 926 | + EXPECT_TRUE(result.second_eof); |
| 927 | +} |
| 928 | + |
| 929 | +TEST(JsonReaderTest, EmptyFileProjectionPreservesMalformedNullRows) { |
| 930 | + auto result = read_once("empty_projection_malformed.jsonl", "not-json\n{\"id\":1}\n", |
| 931 | + json_scan_params(true, false, "", "", true), {}, {}, true, true); |
| 932 | + ASSERT_TRUE(result.status.ok()) << result.status; |
| 933 | + EXPECT_EQ(result.rows, 2); |
| 934 | + EXPECT_TRUE(result.second_status.ok()); |
| 935 | + EXPECT_TRUE(result.second_eof); |
| 936 | + |
| 937 | + auto strict = read_once("empty_projection_strict.jsonl", "not-json\n", json_scan_params(), {}, |
| 938 | + {}, false, true); |
| 939 | + EXPECT_FALSE(strict.status.ok()); |
| 940 | +} |
| 941 | + |
| 942 | +TEST(JsonReaderTest, EmptyFileProjectionPreservesEmptyInput) { |
| 943 | + for (const std::string content : {"", "\n", "[]\n"}) { |
| 944 | + auto result = read_once("empty_projection_empty.jsonl", content, |
| 945 | + json_scan_params(true, true), {}, {}, false, true); |
| 946 | + ASSERT_TRUE(result.status.ok()) << result.status; |
| 947 | + EXPECT_EQ(result.rows, 0); |
| 948 | + EXPECT_TRUE(result.eof); |
| 949 | + } |
| 950 | +} |
| 951 | + |
| 952 | +TEST(JsonReaderTest, EmptyFileProjectionHonorsBatchSize) { |
| 953 | + const auto path = write_json_file("empty_projection_batches.jsonl", |
| 954 | + "{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n" |
| 955 | + "{\"id\":4}\n{\"id\":5}\n"); |
| 956 | + auto properties = std::make_shared<io::FileSystemProperties>(); |
| 957 | + properties->system_type = TFileType::FILE_LOCAL; |
| 958 | + auto desc = file_description(path.string()); |
| 959 | + auto params = json_scan_params(); |
| 960 | + RuntimeProfile profile("json_empty_projection_batches"); |
| 961 | + MockRuntimeState state; |
| 962 | + state._batch_size = 2; |
| 963 | + JsonReader reader(properties, desc, std::make_shared<io::IOContext>(), &profile, ¶ms, |
| 964 | + file_range(path), {}); |
| 965 | + ASSERT_TRUE(reader.init(&state).ok()); |
| 966 | + ASSERT_TRUE(reader.open(std::make_shared<FileScanRequest>()).ok()); |
| 967 | + for (const size_t expected_rows : {2, 2, 1, 0}) { |
| 968 | + Block block; |
| 969 | + size_t rows = 0; |
| 970 | + bool eof = false; |
| 971 | + ASSERT_TRUE(reader.get_block(&block, &rows, &eof).ok()); |
| 972 | + EXPECT_EQ(block.columns(), 0); |
| 973 | + EXPECT_EQ(rows, expected_rows); |
| 974 | + EXPECT_EQ(eof, expected_rows == 0); |
| 975 | + } |
| 976 | +} |
| 977 | + |
| 978 | +TEST(JsonReaderTest, EmptyFileProjectionMaterializesPartitionRows) { |
| 979 | + const auto path = |
| 980 | + write_json_file("partition_only.jsonl", "{\"id\":1}\n{\"id\":2}\n{\"id\":3}\n"); |
| 981 | + auto params = json_scan_params(); |
| 982 | + const std::vector<SlotDescriptor*> file_slots; |
| 983 | + ColumnDefinition partition; |
| 984 | + partition.identifier = Field::create_field<TYPE_STRING>("dt"); |
| 985 | + partition.local_id = 0; |
| 986 | + partition.name = "dt"; |
| 987 | + partition.type = std::make_shared<DataTypeString>(); |
| 988 | + partition.is_partition_key = true; |
| 989 | + RuntimeProfile profile("json_partition_only"); |
| 990 | + MockRuntimeState state; |
| 991 | + state._batch_size = 2; |
| 992 | + TableReader reader; |
| 993 | + ASSERT_TRUE(reader.init({.projected_columns = {partition}, |
| 994 | + .conjuncts = {}, |
| 995 | + .format = FileFormat::JSON, |
| 996 | + .scan_params = ¶ms, |
| 997 | + .io_ctx = std::make_shared<io::IOContext>(), |
| 998 | + .runtime_state = &state, |
| 999 | + .scanner_profile = &profile, |
| 1000 | + .file_slot_descs = &file_slots}) |
| 1001 | + .ok()); |
| 1002 | + SplitReadOptions split; |
| 1003 | + split.current_range = file_range(path); |
| 1004 | + split.current_split_format = FileFormat::JSON; |
| 1005 | + split.partition_values.emplace("dt", Field::create_field<TYPE_STRING>("p1")); |
| 1006 | + ASSERT_TRUE(reader.prepare_split(split).ok()); |
| 1007 | + bool eos = false; |
| 1008 | + for (const size_t expected_rows : {2, 1, 0}) { |
| 1009 | + Block block({{partition.type->create_column(), partition.type, partition.name}}); |
| 1010 | + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); |
| 1011 | + ASSERT_EQ(block.rows(), expected_rows); |
| 1012 | + EXPECT_EQ(eos, expected_rows == 0); |
| 1013 | + const auto column = block.get_by_position(0).column->convert_to_full_column_if_const(); |
| 1014 | + for (size_t row = 0; row < expected_rows; ++row) { |
| 1015 | + EXPECT_EQ(column->get_data_at(row).to_string(), "p1"); |
| 1016 | + } |
| 1017 | + } |
| 1018 | + EXPECT_TRUE(reader.close().ok()); |
| 1019 | +} |
| 1020 | + |
859 | 1021 | } // namespace doris::format::json |
0 commit comments