Skip to content

Commit fe7c916

Browse files
RaphDalRaphDal
authored
feat(core): add data_id with current_data_id() SQL function (#5917)
1 parent 2b00422 commit fe7c916

9 files changed

Lines changed: 311 additions & 5 deletions

File tree

‎core/src/main/java/io/questdb/cairo/CairoEngine.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,7 @@ public class CairoEngine implements Closeable, WriterSource {
146146
private final DatabaseCheckpointAgent checkpointAgent;
147147
private final CopyContext copyContext;
148148
private final ConcurrentHashMap<TableToken> createTableLock = new ConcurrentHashMap<>();
149+
private final DataID dataID;
149150
private final EngineMaintenanceJob engineMaintenanceJob;
150151
private final FunctionFactoryCache ffCache;
151152
private final MatViewGraph matViewGraph;
@@ -209,6 +210,7 @@ public CairoEngine(CairoConfiguration configuration) {
209210
this.matViewTimerQueue = createMatViewTimerQueue();
210211
this.matViewGraph = new MatViewGraph();
211212
this.frameFactory = new FrameFactory(configuration);
213+
this.dataID = DataID.open(configuration);
212214

213215
settingsStore = new SettingsStore(configuration);
214216
settingsStore.init();
@@ -661,6 +663,10 @@ public CopyContext getCopyContext() {
661663
return copyContext;
662664
}
663665

666+
public DataID getDataID() {
667+
return dataID;
668+
}
669+
664670
public @NotNull DdlListener getDdlListener(TableToken tableToken) {
665671
return tableFlagResolver.isSystem(tableToken.getTableName()) ? DefaultDdlListener.INSTANCE : ddlListener;
666672
}

‎core/src/main/java/io/questdb/cairo/DataID.java‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/*******************************************************************************
2+
* ___ _ ____ ____
3+
* / _ \ _ _ ___ ___| |_| _ \| __ )
4+
* | | | | | | |/ _ \/ __| __| | | | _ \
5+
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
6+
* \__\_\\__,_|\___||___/\__|____/|____/
7+
*
8+
* Copyright (c) 2014-2019 Appsicle
9+
* Copyright (c) 2019-2024 QuestDB
10+
*
11+
* Licensed under the Apache License, Version 2.0 (the "License");
12+
* you may not use this file except in compliance with the License.
13+
* You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*
23+
******************************************************************************/
24+
25+
package io.questdb.cairo;
26+
27+
import io.questdb.cairo.vm.MemoryCMARWImpl;
28+
import io.questdb.std.FilesFacade;
29+
import io.questdb.std.MemoryTag;
30+
import io.questdb.std.Numbers;
31+
import io.questdb.std.Uuid;
32+
import io.questdb.std.str.CharSink;
33+
import io.questdb.std.str.Path;
34+
import io.questdb.std.str.Sinkable;
35+
import org.jetbrains.annotations.NotNull;
36+
37+
/**
38+
* DataID handles the mapping of the .data_id file located at the root of the database.
39+
* Its role is to store a unique id (consisting of a randomly generated 128-bit UUID)
40+
* to uniquely "tag" a `db` directory so the contained tables can be uniquely identified
41+
* across backups and enterprise replication.
42+
* <p>
43+
* One shouldn't modify the data id in an unblank database as it may cause data loss.
44+
* </p>
45+
*/
46+
public final class DataID implements Sinkable {
47+
48+
/**
49+
* The file that contains the serialized DataID value has a name that starts with a `.`
50+
* as this avoids a name clash with a potentially valid table name.
51+
*/
52+
public static final CharSequence FILENAME = ".data_id";
53+
public static long FILE_SIZE = Long.SIZE * 2; // Storing UUID as binary
54+
private final CairoConfiguration configuration;
55+
private final Uuid id;
56+
57+
public DataID(CairoConfiguration configuration, Uuid id) {
58+
this.id = id;
59+
this.configuration = configuration;
60+
}
61+
62+
/**
63+
* Read the `.data_id` file (or creates it if it doesn't exist yet with zero value) and returns its current value.
64+
*
65+
* @param configuration the configuration that is used to provide the FileFacade and DbRoot.
66+
* @return a new data id instance.
67+
*/
68+
public static DataID open(CairoConfiguration configuration) throws CairoException {
69+
long lo, hi;
70+
71+
// N.B.: This will create a new empty file of null `FILE_SIZE` bytes if it doesn't exist.
72+
try (MemoryCMARWImpl mem = openDataIDFile(configuration)) {
73+
lo = mem.getLong(0);
74+
hi = mem.getLong(Long.BYTES);
75+
76+
// If we've just created the file, it will still have empty bytes.
77+
// We need to represent this as a null UUID - our "uninitialized" state.
78+
if ((lo == 0) && (hi == 0)) {
79+
lo = hi = Numbers.LONG_NULL;
80+
}
81+
}
82+
return new DataID(configuration, new Uuid(lo, hi));
83+
}
84+
85+
public long getHi() {
86+
return id.getHi();
87+
}
88+
89+
public long getLo() {
90+
return id.getLo();
91+
}
92+
93+
/**
94+
* Returns whether the data id has been initialized or not.
95+
*
96+
* @return true if the data id is initialized.
97+
*/
98+
public boolean isInitialized() {
99+
return !Uuid.isNull(id.getLo(), id.getHi());
100+
}
101+
102+
/**
103+
* Set the data id to a new value and writes it to `.data_id`.
104+
* This function should be used with care as it may lead to data losses from restore/replication.
105+
*
106+
* @param lo The low bits of the UUID value
107+
* @param hi The high bits of the UUID value
108+
*/
109+
public void set(long lo, long hi) {
110+
if ((lo == id.getLo()) && (hi == id.getHi())) {
111+
return;
112+
}
113+
try (MemoryCMARWImpl mem = openDataIDFile(configuration)) {
114+
mem.putLong(lo);
115+
mem.putLong(hi);
116+
mem.sync(false);
117+
}
118+
this.id.of(lo, hi);
119+
}
120+
121+
@Override
122+
public void toSink(@NotNull CharSink<?> sink) {
123+
id.toSink(sink);
124+
}
125+
126+
private static MemoryCMARWImpl openDataIDFile(CairoConfiguration configuration) {
127+
try (Path path = new Path()) {
128+
path.of(configuration.getDbRoot());
129+
path.concat(FILENAME);
130+
131+
final FilesFacade ff = configuration.getFilesFacade();
132+
return new MemoryCMARWImpl(ff, path.$(), FILE_SIZE, -1, MemoryTag.MMAP_DEFAULT, configuration.getWriterFileOpenOpts());
133+
}
134+
}
135+
}

‎core/src/main/java/io/questdb/griffin/engine/functions/catalogue/CurrentDataIdFunctionFactory.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*******************************************************************************
2+
* ___ _ ____ ____
3+
* / _ \ _ _ ___ ___| |_| _ \| __ )
4+
* | | | | | | |/ _ \/ __| __| | | | _ \
5+
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
6+
* \__\_\\__,_|\___||___/\__|____/|____/
7+
*
8+
* Copyright (c) 2014-2019 Appsicle
9+
* Copyright (c) 2019-2024 QuestDB
10+
*
11+
* Licensed under the Apache License, Version 2.0 (the "License");
12+
* you may not use this file except in compliance with the License.
13+
* You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*
23+
******************************************************************************/
24+
25+
package io.questdb.griffin.engine.functions.catalogue;
26+
27+
import io.questdb.cairo.CairoConfiguration;
28+
import io.questdb.cairo.DataID;
29+
import io.questdb.cairo.sql.Function;
30+
import io.questdb.griffin.FunctionFactory;
31+
import io.questdb.griffin.SqlExecutionContext;
32+
import io.questdb.griffin.engine.functions.constants.UuidConstant;
33+
import io.questdb.std.IntList;
34+
import io.questdb.std.ObjList;
35+
36+
public class CurrentDataIdFunctionFactory implements FunctionFactory {
37+
@Override
38+
public String getSignature() {
39+
return "current_data_id()";
40+
}
41+
42+
@Override
43+
public boolean isRuntimeConstant() {
44+
return true;
45+
}
46+
47+
@Override
48+
public Function newInstance(int position, ObjList<Function> args, IntList argPositions, CairoConfiguration configuration, SqlExecutionContext sqlExecutionContext) {
49+
DataID id = sqlExecutionContext.getCairoEngine().getDataID();
50+
return new UuidConstant(id.getLo(), id.getHi());
51+
}
52+
}

‎core/src/test/java/io/questdb/test/cairo/DataIDTest.java‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*******************************************************************************
2+
* ___ _ ____ ____
3+
* / _ \ _ _ ___ ___| |_| _ \| __ )
4+
* | | | | | | |/ _ \/ __| __| | | | _ \
5+
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
6+
* \__\_\\__,_|\___||___/\__|____/|____/
7+
*
8+
* Copyright (c) 2014-2019 Appsicle
9+
* Copyright (c) 2019-2024 QuestDB
10+
*
11+
* Licensed under the Apache License, Version 2.0 (the "License");
12+
* you may not use this file except in compliance with the License.
13+
* You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*
23+
******************************************************************************/
24+
25+
package io.questdb.test.cairo;
26+
27+
import io.questdb.cairo.DataID;
28+
import io.questdb.std.Long256;
29+
import io.questdb.std.Long256Impl;
30+
import io.questdb.std.Numbers;
31+
import io.questdb.std.Rnd;
32+
import io.questdb.std.Uuid;
33+
import io.questdb.test.AbstractCairoTest;
34+
import org.junit.Assert;
35+
import org.junit.Test;
36+
37+
public class DataIDTest extends AbstractCairoTest {
38+
@Test
39+
public void testOpenDataID() throws Exception {
40+
assertMemoryLeak(() -> {
41+
DataID id = DataID.open(configuration);
42+
Assert.assertNotNull(id);
43+
Assert.assertFalse(id.isInitialized());
44+
Assert.assertEquals(Numbers.LONG_NULL, id.getLo());
45+
Assert.assertEquals(Numbers.LONG_NULL, id.getHi());
46+
47+
Rnd rnd = new Rnd(configuration.getMicrosecondClock().getTicks(), configuration.getMillisecondClock().getTicks());
48+
Uuid currentId = new Uuid();
49+
currentId.of(rnd.nextLong(), rnd.nextLong());
50+
id.set(currentId.getLo(), currentId.getHi());
51+
Assert.assertTrue(id.isInitialized());
52+
Assert.assertEquals(id.getLo(), currentId.getLo());
53+
Assert.assertEquals(id.getHi(), currentId.getHi());
54+
55+
DataID updatedId = DataID.open(configuration);
56+
Assert.assertTrue(updatedId.isInitialized());
57+
Assert.assertEquals(updatedId.getLo(), currentId.getLo());
58+
Assert.assertEquals(updatedId.getHi(), currentId.getHi());
59+
});
60+
}
61+
}

‎core/src/test/java/io/questdb/test/cairo/o3/O3FailureTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -599,7 +599,7 @@ public void testFailOnTruncateKeyIndexContended() throws Exception {
599599
@Override
600600
public boolean truncate(long fd, long size) {
601601
// First two calls to truncate are for varchar column
602-
if (size == 0 && counter.getAndIncrement() == 2) {
602+
if (size == 0 && counter.getAndIncrement() == 3) {
603603
return false;
604604
}
605605
return super.truncate(fd, size);
@@ -618,7 +618,7 @@ public void testFailOnTruncateKeyValueContended() throws Exception {
618618
@Override
619619
public boolean truncate(long fd, long size) {
620620
// First two calls to truncate are for varchar column
621-
if (size == 0 && counter.getAndIncrement() == 2) {
621+
if (size == 0 && counter.getAndIncrement() == 3) {
622622
return false;
623623
}
624624
return super.truncate(fd, size);

‎core/src/test/java/io/questdb/test/cutlass/http/HttpErrorHandlingTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ public ServerConfiguration getServerConfiguration(Bootstrap bootstrap) throws Ex
7979
new FilesFacadeImpl() {
8080
@Override
8181
public long openRW(LPSZ name, long opts) {
82-
if (counter.incrementAndGet() > 76) {
82+
if (counter.incrementAndGet() > 78) {
8383
throw new RuntimeException("Test error");
8484
}
8585
return super.openRW(name, opts);

‎core/src/test/java/io/questdb/test/cutlass/pgwire/PGErrorHandlingTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,7 @@ public ServerConfiguration getServerConfiguration(Bootstrap bootstrap) throws Ex
9898
new FilesFacadeImpl() {
9999
@Override
100100
public long openRW(LPSZ name, long opts) {
101-
if (counter.incrementAndGet() > 76) {
101+
if (counter.incrementAndGet() > 78) {
102102
throw new RuntimeException("Test error");
103103
}
104104
return super.openRW(name, opts);

‎core/src/test/java/io/questdb/test/griffin/engine/functions/catalogue/CurrentDataIDFunctionFactoryTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*******************************************************************************
2+
* ___ _ ____ ____
3+
* / _ \ _ _ ___ ___| |_| _ \| __ )
4+
* | | | | | | |/ _ \/ __| __| | | | _ \
5+
* | |_| | |_| | __/\__ \ |_| |_| | |_) |
6+
* \__\_\\__,_|\___||___/\__|____/|____/
7+
*
8+
* Copyright (c) 2014-2019 Appsicle
9+
* Copyright (c) 2019-2024 QuestDB
10+
*
11+
* Licensed under the Apache License, Version 2.0 (the "License");
12+
* you may not use this file except in compliance with the License.
13+
* You may obtain a copy of the License at
14+
*
15+
* http://www.apache.org/licenses/LICENSE-2.0
16+
*
17+
* Unless required by applicable law or agreed to in writing, software
18+
* distributed under the License is distributed on an "AS IS" BASIS,
19+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20+
* See the License for the specific language governing permissions and
21+
* limitations under the License.
22+
*
23+
******************************************************************************/
24+
25+
package io.questdb.test.griffin.engine.functions.catalogue;
26+
27+
import io.questdb.std.Rnd;
28+
import io.questdb.std.Uuid;
29+
import io.questdb.test.AbstractCairoTest;
30+
import org.junit.Test;
31+
32+
public class CurrentDataIDFunctionFactoryTest extends AbstractCairoTest {
33+
34+
@Test
35+
public void testUninitializedDataID() throws Exception {
36+
assertSql("current_data_id\n\n",
37+
"select current_data_id();");
38+
}
39+
40+
@Test
41+
public void testSetDataID() throws Exception {
42+
final Uuid newID = new Uuid();
43+
Rnd rnd = configuration.getRandom();
44+
newID.of(rnd.nextLong(), rnd.nextLong());
45+
sink.clear();
46+
newID.toSink(sink);
47+
engine.getDataID().set(newID.getLo(), newID.getHi());
48+
final String id = sink.toString();
49+
assertSql("current_data_id\n" + id + "\n",
50+
"select current_data_id();");
51+
}
52+
}

‎core/src/test/java/io/questdb/test/griffin/engine/groupby/SampleByTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7796,7 +7796,7 @@ public void testSampleFillLinearConstructorFail() throws Exception {
77967796
") timestamp(k) partition by NONE"
77977797
);
77987798

7799-
CairoConfiguration configuration = createMmapFailingConfiguration(4);
7799+
CairoConfiguration configuration = createMmapFailingConfiguration(5);
78007800

78017801
try (CairoEngine engine = new CairoEngine(configuration)) {
78027802
try (SqlCompiler compiler = engine.getSqlCompiler()) {

0 commit comments

Comments
 (0)