Skip to content

Commit 647edef

Browse files
bluestreak01bluestreak01
committed
Merge remote-tracking branch 'origin/master' into vi_sql_validation
2 parents fb5d2f3 + c6ee66e commit 647edef

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+1595
-465
lines changed

‎benchmarks/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
<groupId>io.questdb</groupId>
2929
<artifactId>benchmarks</artifactId>
30-
<version>9.1.2-SNAPSHOT</version>
30+
<version>9.2.1-SNAPSHOT</version>
3131
<packaging>jar</packaging>
3232
<name>JMH benchmarks for QuestDB</name>
3333

‎benchmarks/src/main/java/org/questdb/WalWriterSymbolBenchmark.java‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
/*******************************************************************************
2+
* ___ _ ____ ____
3+
* / _ \ _ _ ___ ___| |_| _ \| __ )
4+
* | | | | | | |/ _ \/ __| __| | | | _ \
5+
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
6+
* \__\_\\__,_|\___||___/\__|____/|____/
7+
*
8+
* Copyright (c) 2014-2019 Appsicle
9+
* Copyright (c) 2019-2024 QuestDB
10+
*
11+
* Licensed under the Apache License, Version 2.0 (the "License");
12+
* you may not use this file except in compliance with the License.
13+
* You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*
23+
******************************************************************************/
24+
25+
package org.questdb;
26+
27+
import io.questdb.cairo.sql.SymbolTable;
28+
import io.questdb.cairo.vm.MemoryCARWImpl;
29+
import io.questdb.cairo.vm.api.MemoryARW;
30+
import io.questdb.cairo.wal.DirectCharSequenceIntHashMap;
31+
import io.questdb.std.CharSequenceIntHashMap;
32+
import io.questdb.std.Chars;
33+
import io.questdb.std.MemoryTag;
34+
import io.questdb.std.Numbers;
35+
import io.questdb.std.Rnd;
36+
import io.questdb.std.Unsafe;
37+
import io.questdb.std.str.DirectString;
38+
import org.openjdk.jmh.annotations.Benchmark;
39+
import org.openjdk.jmh.annotations.BenchmarkMode;
40+
import org.openjdk.jmh.annotations.Level;
41+
import org.openjdk.jmh.annotations.Mode;
42+
import org.openjdk.jmh.annotations.OutputTimeUnit;
43+
import org.openjdk.jmh.annotations.Scope;
44+
import org.openjdk.jmh.annotations.Setup;
45+
import org.openjdk.jmh.annotations.State;
46+
import org.openjdk.jmh.annotations.TearDown;
47+
import org.openjdk.jmh.profile.GCProfiler;
48+
import org.openjdk.jmh.runner.Runner;
49+
import org.openjdk.jmh.runner.RunnerException;
50+
import org.openjdk.jmh.runner.options.Options;
51+
import org.openjdk.jmh.runner.options.OptionsBuilder;
52+
53+
import java.util.concurrent.TimeUnit;
54+
55+
@State(Scope.Thread)
56+
@BenchmarkMode(Mode.AverageTime)
57+
@OutputTimeUnit(TimeUnit.NANOSECONDS)
58+
// This benchmark compares the speed of some hash maps implementation for the Wal Writer Symbols map.
59+
// In this use case, we are only appending new keys to the map, and at tx completion we are clearing the whole
60+
// map (thus the .clear in each Invocation).
61+
public class WalWriterSymbolBenchmark {
62+
private static final double loadFactor = 0.7;
63+
private final CharSequenceIntHashMap hmap = new CharSequenceIntHashMap(State.nSymbols, loadFactor, SymbolTable.VALUE_NOT_FOUND);
64+
private final DirectCharSequenceIntHashMap symbolHashMap = new DirectCharSequenceIntHashMap(State.nSymbols, loadFactor, SymbolTable.VALUE_NOT_FOUND);
65+
66+
public static void main(String[] args) throws RunnerException {
67+
Options opt = new OptionsBuilder()
68+
.include(WalWriterSymbolBenchmark.class.getSimpleName())
69+
.warmupIterations(2)
70+
.measurementIterations(3)
71+
.forks(0)
72+
.addProfiler(GCProfiler.class)
73+
.build();
74+
75+
new Runner(opt).run();
76+
}
77+
78+
@Benchmark
79+
public void CharSequenceIntHashMap(State state) {
80+
for (int i = 0; i < state.indices.length; i++) {
81+
CharSequence key = state.symbols[state.indices[i]];
82+
final int index = hmap.keyIndex(key);
83+
if (index > -1) {
84+
hmap.putAt(index, key, i);
85+
}
86+
}
87+
88+
int symbolCount = 0;
89+
long appendOffset = state.mem.getAppendOffset();
90+
state.mem.putInt(0);
91+
92+
for (int j = 0, n = hmap.size(); j < n; j++) {
93+
final CharSequence symbol = hmap.keys().getQuick(j);
94+
assert symbol != null;
95+
final int value = hmap.get(symbol);
96+
// Ignore symbols cached from symbolMapReader
97+
if (value >= state.minimumValue) {
98+
state.mem.putInt(value);
99+
state.mem.putStr(symbol);
100+
symbolCount++;
101+
}
102+
}
103+
state.mem.putInt(appendOffset, symbolCount);
104+
105+
hmap.clear();
106+
}
107+
108+
@Benchmark
109+
public void DirectCharSequenceIntHashMap(State state) {
110+
for (int i = 0; i < state.indices.length; i++) {
111+
CharSequence key = state.symbols[state.indices[i]];
112+
final int hashCode = Chars.hashCode(key);
113+
final int index = symbolHashMap.keyIndex(key, hashCode);
114+
if (index > -1) {
115+
symbolHashMap.putAt(index, key, i, hashCode);
116+
}
117+
}
118+
119+
long appendOffset = state.mem.getAppendOffset();
120+
state.mem.putInt(0);
121+
122+
int copied = symbolHashMap.copyTo(state.mem, state.minimumValue);
123+
124+
state.mem.putInt(appendOffset, copied);
125+
126+
symbolHashMap.clear();
127+
}
128+
129+
@org.openjdk.jmh.annotations.State(Scope.Thread)
130+
public static class State {
131+
public static final int avgReadPerSymbol = 10;
132+
public static final int nSymbols = 1000;
133+
public static final int symbolLength = 32;
134+
public final MemoryARW mem;
135+
public final int minimumValue;
136+
private final Rnd rnd;
137+
public int[] indices;
138+
public CharSequence[] symbols;
139+
private long lo;
140+
141+
public State() {
142+
this.rnd = new Rnd();
143+
this.indices = new int[nSymbols * avgReadPerSymbol];
144+
this.symbols = new CharSequence[nSymbols];
145+
this.mem = new MemoryCARWImpl(Numbers.ceilPow2(4 + nSymbols * ((symbolLength << 1) + 8)), 1, MemoryTag.NATIVE_DEFAULT);
146+
this.minimumValue = nSymbols / 2;
147+
}
148+
149+
@Setup(Level.Trial)
150+
public void setup() {
151+
// To have an apple-to-apple comparison, we cannot rely on String
152+
// instead we're using off-heap memory to store the Strings.
153+
this.lo = Unsafe.malloc(nSymbols * symbolLength * 2, MemoryTag.NATIVE_DEFAULT);
154+
155+
for (int i = 0; i < nSymbols; i++) {
156+
CharSequence symbol = rnd.nextChars(symbolLength);
157+
for (int j = 0; j < symbolLength; j++) {
158+
Unsafe.getUnsafe().putChar(lo + (i * symbolLength + j) * 2, symbol.charAt(j));
159+
}
160+
DirectString ds = new DirectString();
161+
ds.of(lo + (i * symbolLength) * 2, symbolLength);
162+
symbols[i] = ds;
163+
}
164+
165+
for (int i = 0; i < nSymbols * avgReadPerSymbol; i++) {
166+
indices[i] = rnd.nextInt(nSymbols);
167+
}
168+
}
169+
170+
@Setup(Level.Invocation)
171+
public void setupInvocation() {
172+
mem.truncate();
173+
}
174+
175+
@TearDown(Level.Trial)
176+
public void tearDown() {
177+
if (this.lo != 0) {
178+
Unsafe.free(lo, nSymbols * symbolLength * 2, MemoryTag.NATIVE_DEFAULT);
179+
this.lo = 0;
180+
}
181+
mem.close();
182+
}
183+
}
184+
}

‎ci/github-release-pipeline.yml‎

Lines changed: 74 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,10 @@ stages:
2020
imageName: 'ubuntu-latest'
2121
poolName: "Azure Pipelines"
2222
os: Linux
23-
jdk: "1.17"
2423
windows:
2524
imageName: "windows-latest"
2625
poolName: "Azure Pipelines"
2726
os: Windows
28-
jdk: "1.17"
2927
pool:
3028
vmImage: $(imageName)
3129
name: $(poolName)
@@ -35,19 +33,90 @@ stages:
3533
fetchDepth: 1
3634
lfs: false
3735
submodules: true
36+
37+
# --- GraalVM CE 17.0.9 install + SHA256 verify (Linux) ---
38+
- bash: |
39+
set -eux
40+
41+
GRAAL_VERSION=17.0.9
42+
TAG="jdk-${GRAAL_VERSION}"
43+
FILENAME="graalvm-community-jdk-${GRAAL_VERSION}_linux-x64_bin.tar.gz"
44+
BASE_URL="https://github.com/graalvm/graalvm-ce-builds/releases/download/${TAG}"
45+
46+
# Download archive and checksum
47+
curl -L -o "${FILENAME}" "${BASE_URL}/${FILENAME}"
48+
curl -L -o "${FILENAME}.sha256" "${BASE_URL}/${FILENAME}.sha256"
49+
50+
# .sha256 only contains the hash → wrap with filename for sha256sum -c
51+
HASH="$(cut -d ' ' -f1 "${FILENAME}.sha256")"
52+
echo "${HASH} ${FILENAME}" | sha256sum -c -
53+
54+
# Install to /usr/lib/jvm/graalvm-community-jdk-${GRAAL_VERSION}
55+
INSTALL_DIR="/usr/lib/jvm/graalvm-community-jdk-${GRAAL_VERSION}"
56+
mkdir -p "${INSTALL_DIR}"
57+
tar -xzf "${FILENAME}" -C "${INSTALL_DIR}" --strip-components=1
58+
59+
echo "##vso[task.setvariable variable=GRAALVM_HOME]${INSTALL_DIR}"
60+
displayName: "Install GraalVM Community 17.0.9 (Linux, SHA256 verified)"
61+
condition: eq( variables['os'], 'Linux')
62+
63+
# --- GraalVM CE 17.0.9 install + SHA256 verify (Windows) ---
64+
- powershell: |
65+
$ErrorActionPreference = "Stop"
66+
67+
$GRAAL_VERSION = "17.0.9"
68+
$tag = "jdk-$GRAAL_VERSION"
69+
$fileName = "graalvm-community-jdk-$GRAAL_VERSION`_windows-x64_bin.zip"
70+
$baseUrl = "https://github.com/graalvm/graalvm-ce-builds/releases/download/$tag"
71+
$zipPath = Join-Path $env:USERPROFILE $fileName
72+
$shaPath = "$zipPath.sha256"
73+
74+
# Download archive and checksum
75+
Write-Host "Downloading $baseUrl/$fileName"
76+
Invoke-WebRequest "$baseUrl/$fileName" -OutFile $zipPath
77+
78+
Write-Host "Downloading $baseUrl/$fileName.sha256"
79+
Invoke-WebRequest "$baseUrl/$fileName.sha256" -OutFile $shaPath
80+
81+
# Verify checksum (file contains only the hash)
82+
$expected = (Get-Content $shaPath).Split(" ")[0].ToLower()
83+
$actual = (Get-FileHash $zipPath -Algorithm SHA256).Hash.ToLower()
84+
85+
if ($expected -ne $actual) {
86+
throw "SHA256 mismatch for $fileName. Expected $expected but got $actual"
87+
} else {
88+
Write-Host "SHA256 checksum OK for $fileName"
89+
}
90+
91+
# Extract
92+
$dest = Join-Path $env:USERPROFILE "graalvm-ce-17"
93+
if (Test-Path $dest) { Remove-Item $dest -Recurse -Force }
94+
Expand-Archive $zipPath -DestinationPath $dest
95+
96+
# The zip contains a top-level directory; grab it as GRAALVM_HOME
97+
$dir = Get-ChildItem $dest | Where-Object { $_.PSIsContainer } | Select-Object -First 1
98+
$graalHome = $dir.FullName
99+
100+
Write-Host "GRAALVM_HOME = $graalHome"
101+
Write-Host "##vso[task.setvariable variable=GRAALVM_HOME]$graalHome"
102+
displayName: "Install GraalVM Community 17.0.9 (Windows, SHA256 verified)"
103+
condition: eq( variables['os'], 'Windows')
104+
38105
- task: Maven@3
39106
displayName: "Building QuestDB"
40107
inputs:
41108
mavenPomFile: "core/pom.xml"
42109
goals: "package"
43-
options:
44-
"-DskipTests -Dhttp.keepAlive=false -P build-web-console,build-binaries"
45-
jdkVersionOption: $(jdk)
110+
options: "-DskipTests -Dhttp.keepAlive=false -P build-web-console,build-binaries"
111+
javaHomeOption: 'Path'
112+
jdkDirectory: '$(GRAALVM_HOME)'
113+
46114
- task: CopyFiles@2
47115
inputs:
48116
SourceFolder: '$(build.sourcesDirectory)/core/target'
49117
Contents: '$(build.sourcesDirectory)/core/target/questdb-*.gz'
50118
TargetFolder: '$(build.artifactstagingdirectory)'
119+
51120
- task: PublishBuildArtifacts@1
52121
inputs:
53122
PathtoPublish: '$(Build.ArtifactStagingDirectory)'

‎ci/templates/self-hosted-jobs.yml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,3 +46,27 @@ jobs:
4646
JAVA_HOME: "/usr/lib/jvm/java-17-openjdk-amd64"
4747
steps:
4848
- template: steps.yml
49+
50+
- job: linux_x64_graal
51+
workspace:
52+
clean: all
53+
displayName: "on linux-x86-graal"
54+
pool:
55+
name: "hetzner-docker-graal"
56+
timeoutInMinutes: 60
57+
condition: eq(variables['System.PullRequest.IsFork'], 'false')
58+
variables:
59+
SOURCE_CODE_CHANGED: $[stageDependencies.CheckChanges.CheckChanges.outputs['check_coverage.SOURCE_CODE_CHANGED']]
60+
RUST_SOURCE_CODE_CHANGED: $[stageDependencies.CheckChanges.CheckChanges.outputs['check_coverage.RUST_SOURCE_CODE_CHANGED']]
61+
CODE_COVERAGE_TOOL_OPTION: $[stageDependencies.CheckChanges.CheckChanges.outputs['check_coverage.CODE_COVERAGE_TOOL_OPTION']]
62+
DIFF_COVER_THRESHOLD_PCT: 50
63+
ARCHIVED_CRASH_LOG: "$(Build.ArtifactStagingDirectory)/questdb-crash-$(Build.SourceBranchName)-$(Build.SourceVersion)-$(System.StageAttempt)-$(Agent.OS)-$(jdk).log"
64+
testset: "all"
65+
javadoc_step: ""
66+
javadoc_profile: ""
67+
jdk: "1.17"
68+
JAVA_HOME_17_X64: "/opt/graalvm"
69+
JAVA_HOME: "/opt/graalvm"
70+
os: Linux
71+
steps:
72+
- template: steps.yml

‎ci/templates/steps.yml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,14 @@ steps:
3333
displayName: "Install JDK and Maven"
3434
condition: |
3535
eq(variables['poolName'], 'hetzner-docker-arm')
36+
- bash: |
37+
JAVA_PATH="/opt/graalvm"
38+
echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_PATH"
39+
echo "##vso[task.setvariable variable=JAVA_HOME_17_X64]$JAVA_PATH"
40+
echo "JAVA_HOME will be set to: $JAVA_PATH"
41+
displayName: "Install JDK and Maven"
42+
condition: |
43+
eq(variables['poolName'], 'hetzner-docker-graal')
3644
- task: Cache@2
3745
displayName: "Cache non-default Maven"
3846
continueOnError: true

‎ci/test-pipeline.yml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,14 +26,24 @@ stages:
2626
- template: templates/check-changes-job.yml
2727

2828
- stage: SelfHostedRunGriffin
29-
displayName: "SelfHosted Griffin And Fuzz tests"
29+
displayName: "SelfHosted Griffin tests"
3030
condition: eq(variables['System.PullRequest.IsFork'], 'false')
3131
dependsOn:
3232
- CheckChanges
3333
jobs:
3434
- template: templates/self-hosted-jobs.yml
3535
variables:
36-
includeTests: "**/griffin/**,**/cairo/**"
36+
includeTests: "**/griffin/**"
37+
38+
- stage: SelfHostedRunCairo
39+
displayName: "SelfHosted Cairo tests"
40+
condition: eq(variables['System.PullRequest.IsFork'], 'false')
41+
dependsOn:
42+
- CheckChanges
43+
jobs:
44+
- template: templates/self-hosted-jobs.yml
45+
variables:
46+
includeTests: "**/cairo/**"
3747

3848
- stage: SelfHostedRunOther
3949
displayName: "SelfHosted Other tests"

‎compat/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727

2828
<groupId>io.questdb</groupId>
2929
<artifactId>compat</artifactId>
30-
<version>9.1.2-SNAPSHOT</version>
30+
<version>9.2.1-SNAPSHOT</version>
3131

3232
<name>Compatibility tests for QuestDB</name>
3333

0 commit comments

Comments
 (0)