Skip to content

Object(JSON) with ids in keys is unusable (exponential slowdown and memory usage) #39492

@filimonov

Description

@filimonov
SET allow_experimental_object_type = 1;

CREATE TABLE json_test
(
    `raw` JSON
)
ENGINE = MergeTree
ORDER BY tuple()
SETTINGS index_granularity = 8192;

## single row - ok
INSERT INTO json_test SELECT format('{{"bad_subkey":{{"key{}":[{{"value":"{}","bad":true}}]}}}}', toString(number), toString(rand())) FROM numbers(1)

select * from json_test format JSONEachRow settings output_format_json_named_tuples_as_objects = 1;

{"raw":{"bad_subkey":{"key0":[{"bad":1,"value":"3736142820"}]}}}

## up to 1140 rows - slow, but still kinda works
INSERT INTO json_test SELECT format('{{"bad_subkey":{{"key{}":[{{"value":"{}","bad":true}}]}}}}', toString(number), toString(rand())) FROM numbers(1140)

-- MemoryTracker: Peak memory usage (for query): 72.13 MiB.
--- TCPHandler: Processed in 0.797080049 sec.

## from 1141 rows - memory usage suddenly jumps to gigabytes

INSERT INTO json_test SELECT format('{{"bad_subkey":{{"key{}":[{{"value":"{}","bad":true}}]}}}}', toString(number), toString(rand())) FROM numbers(1141)

--- executeQuery: Read 1141 rows, 8.91 KiB in 1.175646399 sec., 970 rows/sec., 7.58 KiB/sec.
---  MemoryTracker: Peak memory usage (for query): 6.78 GiB.

# With more than few thousands rows - unusable completely

Metadata

Metadata

Assignees

No one assigned

    Labels

    experimental featureBug in the feature that should not be used in productionmemoryWhen memory usage is higher than expectednot plannedKnown issue, no plans to fix it currenltyperformance

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions