Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
72 commits
Select commit Hold shift + click to select a range
714570a
Prototype
mtopolnik Oct 30, 2025
3e1a500
Small fix
mtopolnik Oct 30, 2025
98776a7
Modified hashcode
mtopolnik Oct 30, 2025
ffa59a8
bugfix, integration, test
bluestreak01 Oct 30, 2025
1835239
Merge branch 'master' into mt_asof-linear-symbol
bluestreak01 Oct 31, 2025
8f3e640
remove debris
bluestreak01 Oct 31, 2025
8e29a58
fixes
bluestreak01 Oct 31, 2025
fa6710d
Merge remote-tracking branch 'origin/master' into mt_asof-linear-symbol
bluestreak01 Oct 31, 2025
38a1926
Merge branch 'master' into mt_asof-linear-symbol
bluestreak01 Oct 31, 2025
b65ba09
AsOfJoinDenseRecordCursorFactory
mtopolnik Oct 31, 2025
6a01908
Fix log line
mtopolnik Oct 31, 2025
19a6f62
Simplify expression in Ligt record cursor
mtopolnik Oct 31, 2025
760b58a
Merge branch 'master' into mt_asof-linear-symbol
bluestreak01 Nov 1, 2025
d3b969c
fix bugs found in review
bluestreak01 Nov 1, 2025
d8ed506
oops
bluestreak01 Nov 1, 2025
901fabe
Fix some bugs
mtopolnik Nov 1, 2025
a8c3bb7
Update test
mtopolnik Nov 1, 2025
5eceb55
Fix some bugs
mtopolnik Nov 2, 2025
acaebfc
Fix AsOfJoinFuzzTest bug
mtopolnik Nov 2, 2025
ae54424
Fix bug
mtopolnik Nov 2, 2025
55625e8
Merge remote-tracking branch 'origin/mt_asof-linear-symbol' into mt_a…
mtopolnik Nov 2, 2025
63f49d3
Revert default ASOF JOIN to Fast
mtopolnik Nov 3, 2025
1db9317
Merge branch 'master' into mt_asof-dense
mtopolnik Nov 3, 2025
62d2fe8
Cleanup after merge
mtopolnik Nov 3, 2025
a411ec2
Fix test
mtopolnik Nov 3, 2025
50ed933
Fix test
mtopolnik Nov 3, 2025
f66b22d
Auto-format
mtopolnik Nov 3, 2025
8d5387c
Formatting
mtopolnik Nov 3, 2025
e56e889
Formatting
mtopolnik Nov 3, 2025
274b911
Formatting
mtopolnik Nov 3, 2025
5423ba4
Formatting
mtopolnik Nov 3, 2025
64bccbf
Merge commit '5423ba436384ea532d1bd6c201c46cd5a7c435f0' into mt_asof-…
mtopolnik Nov 3, 2025
9922096
Fix test
mtopolnik Nov 3, 2025
c19aae1
Introduce ASOF_DENSE hint
mtopolnik Nov 3, 2025
36fb327
Javadoc and comments
mtopolnik Nov 3, 2025
e1d5d84
Rename vars
mtopolnik Nov 3, 2025
9b17170
Refactor and clean up ASOF/LT JOIN code
mtopolnik Nov 4, 2025
94d88fa
Fix bug
mtopolnik Nov 4, 2025
d3ae983
Shorten asof_index_search hint to asof_index
mtopolnik Nov 4, 2025
fae5769
Update tests
mtopolnik Nov 4, 2025
ab79aa5
Update tests
mtopolnik Nov 4, 2025
b107126
Change asof_driveby_caching hint
mtopolnik Nov 4, 2025
6ed74bd
Merge branch 'master' into mt_asof-cleanup
mtopolnik Nov 5, 2025
eb6d34b
Move hint detection closer to usage
mtopolnik Nov 5, 2025
57a6ea7
Merge branch 'mt_asof-cleanup' into mt_asof-dense
mtopolnik Nov 5, 2025
2951a82
Let Dense Scan work with any join key
mtopolnik Nov 5, 2025
65e1839
Use cached symbol mapping in Indexed factory
mtopolnik Nov 6, 2025
83c9f66
Rename ...ColumnAccessHelper
mtopolnik Nov 6, 2025
3fa9aba
Split SymbolShortCircuit into two concerns
mtopolnik Nov 6, 2025
d791ffb
recordSink -> recordCopier
mtopolnik Nov 6, 2025
e4daf91
Optimize join key equality check
mtopolnik Nov 6, 2025
3469027
WIP
mtopolnik Nov 7, 2025
874233c
Move native map creation into cursor constructor
mtopolnik Nov 7, 2025
13876f4
Remove unnecessary ignores
mtopolnik Nov 7, 2025
5bdb549
Stronger self-join test
mtopolnik Nov 7, 2025
e853834
Test self-join optimization with time offset
mtopolnik Nov 7, 2025
d7ac53b
Simplify tests using multiline strings
mtopolnik Nov 7, 2025
7abbfa0
Implement Single Symbol Dense Cursor
mtopolnik Nov 7, 2025
b5c1a60
Merge branch 'mt_asof-cleanup' into mt_asof-dense
mtopolnik Nov 7, 2025
9f28713
Let Light cursor set symbolTable on RecordCopier
mtopolnik Nov 7, 2025
52cbacb
Short-circuit Dense cursor when symbol not found
mtopolnik Nov 7, 2025
feb4588
Delete test of deleted cursor
mtopolnik Nov 7, 2025
3903341
Delete Single Symbol Light cursor
mtopolnik Nov 7, 2025
2071edc
Remove extra blank line
mtopolnik Nov 7, 2025
e827978
Merge branch 'master' into mt_asof-dense
mtopolnik Nov 7, 2025
6c4a201
Merge branch 'master' into mt_asof-dense
mtopolnik Nov 7, 2025
d0b772b
Remove "Scan" from cursor names
mtopolnik Nov 10, 2025
2ee73fe
Merge branch 'master' into mt_asof-dense
mtopolnik Nov 10, 2025
d1125a6
Remove "Scan" from cursor names
mtopolnik Nov 10, 2025
6d0ee6a
Simplify test assertion
mtopolnik Nov 10, 2025
8aac540
Add circuit breaker and @NotNull
mtopolnik Nov 10, 2025
32da72c
small cleanup
bluestreak01 Nov 10, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
399 changes: 214 additions & 185 deletions core/src/main/java/io/questdb/griffin/SqlCodeGenerator.java

Large diffs are not rendered by default.

9 changes: 9 additions & 0 deletions core/src/main/java/io/questdb/griffin/SqlHints.java
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,22 @@
import org.jetbrains.annotations.Nullable;

public final class SqlHints {
public static final String ASOF_DENSE_HINT = "asof_dense";
public static final String ASOF_INDEX_HINT = "asof_index";
public static final String ASOF_LINEAR_HINT = "asof_linear";
public static final String ASOF_MEMOIZED_DRIVEBY_HINT = "asof_memoized_driveby";
public static final String ASOF_MEMOIZED_HINT = "asof_memoized";
public static final String ENABLE_PRE_TOUCH_HINT = "enable_pre_touch";
public static final char HINTS_PARAMS_DELIMITER = ' ';

public static boolean hasAsOfDenseHint(
@NotNull QueryModel queryModel,
@Nullable CharSequence tableNameA,
@Nullable CharSequence tableNameB
) {
return hasHintWithParams(queryModel, ASOF_DENSE_HINT, tableNameA, tableNameB);
}

public static boolean hasAsOfIndexHint(
@NotNull QueryModel queryModel,
@Nullable CharSequence tableNameA,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -233,17 +233,8 @@ public static class GenerateSeriesPeriod {

public static boolean isPotentiallyValidUnit(char c) {
return switch (c) {
// nanos
// compatibility
// micros
// millis
// seconds
// minutes
// compatibility
// hours
// days
// weeks
// months
// n:nanos U:micros u:micros-compatibility T:millis s:seconds m:minutes
// h:hours H:hours-compatibility d:days w:weeks M:months y:years
case 'n', 'u', 'U', 'T', 's', 'm', 'h', 'H', 'd', 'w', 'M', 'y' -> true;
default -> false;
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public AbstractKeyedAsOfJoinRecordCursor(
}

@Override
public final boolean hasNext() {
public boolean hasNext() {
// Common master cursor iteration logic
if (isMasterHasNextPending) {
masterHasNext = masterCursor.hasNext();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
/*******************************************************************************
* ___ _ ____ ____
* / _ \ _ _ ___ ___| |_| _ \| __ )
* | | | | | | |/ _ \/ __| __| | | | _ \
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
* \__\_\\__,_|\___||___/\__|____/|____/
*
* Copyright (c) 2014-2019 Appsicle
* Copyright (c) 2019-2024 QuestDB
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
******************************************************************************/

package io.questdb.griffin.engine.join;

import io.questdb.cairo.CairoConfiguration;
import io.questdb.cairo.ColumnTypes;
import io.questdb.cairo.RecordSink;
import io.questdb.cairo.SingleRecordSink;
import io.questdb.cairo.map.Map;
import io.questdb.cairo.map.MapFactory;
import io.questdb.cairo.map.MapKey;
import io.questdb.cairo.sql.Record;
import io.questdb.cairo.sql.RecordCursor;
import io.questdb.cairo.sql.RecordCursorFactory;
import io.questdb.cairo.sql.RecordMetadata;
import io.questdb.cairo.sql.SqlExecutionCircuitBreaker;
import io.questdb.cairo.sql.TimeFrameRecordCursor;
import io.questdb.griffin.PlanSink;
import io.questdb.griffin.model.JoinContext;
import io.questdb.std.MemoryTag;
import io.questdb.std.Misc;
import io.questdb.std.Transient;

public final class AsOfJoinDenseRecordCursorFactory extends AsOfJoinDenseRecordCursorFactoryBase {
private final RecordSink masterKeyCopier;
private final RecordSink slaveKeyCopier;

public AsOfJoinDenseRecordCursorFactory(
CairoConfiguration configuration,
RecordMetadata metadata,
RecordCursorFactory masterFactory,
RecordSink masterKeyCopier,
RecordCursorFactory slaveFactory,
RecordSink slaveKeyCopier,
int columnSplit,
@Transient ColumnTypes keyTypes,
JoinContext joinContext,
long toleranceInterval
) {
super(metadata, masterFactory, slaveFactory, joinContext, toleranceInterval);
this.masterKeyCopier = masterKeyCopier;
this.slaveKeyCopier = slaveKeyCopier;
Map fwdScanKeyToRowId = null;
Map bwdScanKeyToRowId = null;
try {
long maxSinkTargetHeapSize = (long)
configuration.getSqlHashJoinValuePageSize() * configuration.getSqlHashJoinValueMaxPages();
fwdScanKeyToRowId = MapFactory.createUnorderedMap(configuration, keyTypes, TYPES_VALUE);
bwdScanKeyToRowId = MapFactory.createUnorderedMap(configuration, keyTypes, TYPES_VALUE);
this.cursor = new AsOfJoinDenseRecordCursor(
columnSplit,
fwdScanKeyToRowId,
bwdScanKeyToRowId,
NullRecordFactory.getInstance(slaveFactory.getMetadata()),
masterFactory.getMetadata().getTimestampIndex(),
masterFactory.getMetadata().getTimestampType(),
new SingleRecordSink(maxSinkTargetHeapSize, MemoryTag.NATIVE_RECORD_CHAIN),
slaveFactory.getMetadata().getTimestampIndex(),
slaveFactory.getMetadata().getTimestampType(),
new SingleRecordSink(maxSinkTargetHeapSize, MemoryTag.NATIVE_RECORD_CHAIN)
);
} catch (Throwable th) {
Misc.free(bwdScanKeyToRowId);
Misc.free(fwdScanKeyToRowId);
close();
throw th;
}
}

@Override
protected void putFactoryType(PlanSink sink) {
sink.type("AsOf Join Dense");
}

private class AsOfJoinDenseRecordCursor extends AsOfJoinDenseRecordCursorBase {
private final SingleRecordSink masterSinkTarget;
private final SingleRecordSink slaveSinkTarget;

AsOfJoinDenseRecordCursor(
int columnSplit,
Map fwdScanKeyToRowId,
Map bwdScanKeyToRowId,
Record nullRecord,
int masterTimestampIndex,
int masterTimestampType,
SingleRecordSink masterSinkTarget,
int slaveTimestampIndex,
int slaveTimestampType,
SingleRecordSink slaveSinkTarget
) {
super(
columnSplit,
fwdScanKeyToRowId,
bwdScanKeyToRowId,
nullRecord,
masterTimestampIndex,
masterTimestampType,
slaveTimestampIndex,
slaveTimestampType
);
this.masterSinkTarget = masterSinkTarget;
this.slaveSinkTarget = slaveSinkTarget;
}

@Override
public void close() {
Misc.free(slaveSinkTarget);
Misc.free(masterSinkTarget);
super.close();
}

@Override
public void of(RecordCursor masterCursor, TimeFrameRecordCursor slaveCursor, SqlExecutionCircuitBreaker circuitBreaker) {
super.of(masterCursor, slaveCursor, circuitBreaker);
masterSinkTarget.reopen();
slaveSinkTarget.reopen();
}

@Override
protected int getSlaveJoinKey() {
slaveSinkTarget.clear();
slaveKeyCopier.copy(slaveRecB, slaveSinkTarget);
return DUMMY_VALUE;
}

@Override
protected boolean joinKeysMatch(int slaveKeyToFind, int slaveKey) {
return masterSinkTarget.memeq(slaveSinkTarget);
}

@Override
protected void putSlaveJoinKey(MapKey key) {
key.put(slaveRecB, slaveKeyCopier);
}

@Override
protected void putSlaveKeyToFind(MapKey key, int slaveKeyToFind) {
key.put(masterRecord, masterKeyCopier);
}

@Override
protected int setupSymbolKeyToFind() {
masterSinkTarget.clear();
masterKeyCopier.copy(masterRecord, masterSinkTarget);
return DUMMY_VALUE;
}
}
}
Loading
Loading