Skip to content

Commit a4e3adb

Browse files
xiangfu0xiangfu0
authored
Refactor offline quickstarts to load all the tables by default (#9206)
1 parent 2a43d2a commit a4e3adb

16 files changed

+280
-553
lines changed

‎pinot-tools/src/main/java/org/apache/pinot/tools/EmptyQuickstart.java‎

Lines changed: 3 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -18,61 +18,20 @@
1818
*/
1919
package org.apache.pinot.tools;
2020

21-
import java.io.File;
2221
import java.util.ArrayList;
2322
import java.util.Arrays;
2423
import java.util.List;
25-
import org.apache.pinot.spi.auth.AuthProvider;
2624
import org.apache.pinot.tools.admin.PinotAdministrator;
27-
import org.apache.pinot.tools.admin.command.QuickstartRunner;
2825

2926

30-
public class EmptyQuickstart extends QuickStartBase {
27+
public class EmptyQuickstart extends Quickstart {
3128
@Override
3229
public List<String> types() {
3330
return Arrays.asList("EMPTY", "DEFAULT");
3431
}
3532

36-
public AuthProvider getAuthProvider() {
37-
return null;
38-
}
39-
40-
public void execute()
41-
throws Exception {
42-
File quickstartTmpDir = new File(_dataDir.getAbsolutePath());
43-
File dataDir = new File(quickstartTmpDir, "rawdata");
44-
if (!dataDir.mkdirs()) {
45-
printStatus(Quickstart.Color.YELLOW, "***** Bootstrapping data from existing directory *****");
46-
} else {
47-
printStatus(Quickstart.Color.YELLOW, "***** Creating new data directory for fresh installation *****");
48-
}
49-
50-
QuickstartRunner runner =
51-
new QuickstartRunner(new ArrayList<>(), 1, 1, 1, 1,
52-
dataDir, true, getAuthProvider(), getConfigOverrides(), _zkExternalAddress, false);
53-
54-
if (_zkExternalAddress != null) {
55-
printStatus(Quickstart.Color.CYAN, "***** Starting controller, broker and server *****");
56-
} else {
57-
printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
58-
}
59-
60-
runner.startAll();
61-
62-
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
63-
try {
64-
printStatus(Quickstart.Color.GREEN, "***** Shutting down empty quick start *****");
65-
runner.stop();
66-
} catch (Exception e) {
67-
e.printStackTrace();
68-
}
69-
}));
70-
71-
waitForBootstrapToComplete(runner);
72-
73-
printStatus(Quickstart.Color.YELLOW, "***** Empty quickstart setup complete *****");
74-
printStatus(Quickstart.Color.GREEN,
75-
"You can always go to http://localhost:9000 to play around in the query console");
33+
public String[] getDefaultBatchTableDirectories() {
34+
return new String[]{};
7635
}
7736

7837
public static void main(String[] args)

‎pinot-tools/src/main/java/org/apache/pinot/tools/GenericQuickstart.java‎

Lines changed: 20 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,16 @@
1818
*/
1919
package org.apache.pinot.tools;
2020

21-
import com.google.common.base.Preconditions;
22-
import com.google.common.collect.Lists;
23-
import java.io.File;
24-
import java.net.URL;
2521
import java.util.Arrays;
2622
import java.util.List;
27-
import org.apache.commons.io.FileUtils;
2823
import org.apache.pinot.common.utils.ZkStarter;
2924
import org.apache.pinot.spi.stream.StreamDataProvider;
3025
import org.apache.pinot.spi.stream.StreamDataServerStartable;
31-
import org.apache.pinot.tools.Quickstart.Color;
3226
import org.apache.pinot.tools.admin.command.QuickstartRunner;
3327
import org.apache.pinot.tools.utils.KafkaStarterUtils;
3428
import org.slf4j.Logger;
3529
import org.slf4j.LoggerFactory;
3630

37-
import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
38-
3931

4032
/**
4133
* Sets up a demo Pinot cluster with 1 zookeeper, 1 controller, 1 broker and 1 server
@@ -49,29 +41,12 @@
4941
* ingestion_job_spec.json
5042
* </code>
5143
*/
52-
public class GenericQuickstart extends QuickStartBase {
44+
public class GenericQuickstart extends Quickstart {
5345
private static final Logger LOGGER = LoggerFactory.getLogger(GenericQuickstart.class);
54-
private final File _schemaFile;
55-
private final File _tableConfigFile;
56-
private final File _tableDirectory;
57-
private final String _tableName;
5846
private StreamDataServerStartable _kafkaStarter;
5947
private ZkStarter.ZookeeperInstance _zookeeperInstance;
6048

6149
public GenericQuickstart() {
62-
this(GenericQuickstart.class.getClassLoader().getResource("examples/batch/starbucksStores").getPath(),
63-
"starbucksStores");
64-
}
65-
66-
public GenericQuickstart(String tableDirectoryPath, String tableName) {
67-
_tableDirectory = new File(tableDirectoryPath);
68-
_tableName = tableName;
69-
70-
if (!_tableDirectory.exists()) {
71-
Preconditions.checkState(_tableDirectory.mkdirs());
72-
}
73-
_schemaFile = new File(_tableDirectory, "schema.json");
74-
_tableConfigFile = new File(_tableDirectory, "table_config.json");
7550
}
7651

7752
private void startKafka() {
@@ -91,41 +66,9 @@ public List<String> types() {
9166
return Arrays.asList("GENERIC");
9267
}
9368

94-
public void execute()
69+
@Override
70+
public void runSampleQueries(QuickstartRunner runner)
9571
throws Exception {
96-
97-
File tempDir = new File(FileUtils.getTempDirectory(), String.valueOf(System.currentTimeMillis()));
98-
Preconditions.checkState(tempDir.mkdirs());
99-
QuickstartTableRequest request = new QuickstartTableRequest(_tableDirectory.getAbsolutePath());
100-
final QuickstartRunner runner =
101-
new QuickstartRunner(Lists.newArrayList(request), 1, 1, 1, 1, tempDir, getConfigOverrides());
102-
103-
printStatus(Color.CYAN, "***** Starting Kafka *****");
104-
startKafka();
105-
106-
printStatus(Color.CYAN, "***** Starting zookeeper, controller, server and broker *****");
107-
runner.startAll();
108-
109-
printStatus(Color.CYAN, "***** Adding table *****");
110-
runner.bootstrapTable();
111-
112-
printStatus(Color.CYAN, "***** Waiting for 10 seconds for a few events to get populated *****");
113-
Thread.sleep(10000);
114-
115-
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
116-
try {
117-
printStatus(Color.GREEN, "***** Shutting down QuickStart cluster *****");
118-
runner.stop();
119-
_kafkaStarter.stop();
120-
ZkStarter.stopLocalZkServer(_zookeeperInstance);
121-
FileUtils.deleteDirectory(_tableDirectory);
122-
} catch (Exception e) {
123-
LOGGER.error("Caught exception in shutting down QuickStart cluster", e);
124-
}
125-
}));
126-
127-
printStatus(Color.YELLOW, "***** GenericQuickStart demo quickstart setup complete *****");
128-
12972
String q1 = "select count(*) from starbucksStores limit 0";
13073
printStatus(Color.YELLOW, "Total number of documents in the table");
13174
printStatus(Color.CYAN, "Query : " + q1);
@@ -145,17 +88,27 @@ public void execute()
14588
printStatus(Color.CYAN, "Query : " + q3);
14689
printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q3)));
14790
printStatus(Color.GREEN, "***************************************************");
91+
}
92+
93+
public void execute()
94+
throws Exception {
95+
printStatus(Color.CYAN, "***** Starting Kafka *****");
96+
startKafka();
14897

149-
printStatus(Color.GREEN, "You can always go to http://localhost:9000 to play around in the query console");
98+
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
99+
try {
100+
printStatus(Color.GREEN, "***** Shutting down QuickStart cluster *****");
101+
_kafkaStarter.stop();
102+
ZkStarter.stopLocalZkServer(_zookeeperInstance);
103+
} catch (Exception e) {
104+
LOGGER.error("Caught exception in shutting down QuickStart cluster", e);
105+
}
106+
}));
107+
super.execute();
150108
}
151109

152110
public static void main(String[] args)
153111
throws Exception {
154-
ClassLoader classLoader = GenericQuickstart.class.getClassLoader();
155-
URL resource = classLoader.getResource("examples/batch/starbucksStores");
156-
String tableDirectoryPath = resource.getPath();
157-
158-
GenericQuickstart quickstart = new GenericQuickstart(tableDirectoryPath, "starbucksStores");
159-
quickstart.execute();
112+
new GenericQuickstart().execute();
160113
}
161114
}

‎pinot-tools/src/main/java/org/apache/pinot/tools/JoinQuickStart.java‎

Lines changed: 7 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -18,87 +18,24 @@
1818
*/
1919
package org.apache.pinot.tools;
2020

21-
import com.google.common.base.Preconditions;
22-
import com.google.common.collect.Lists;
23-
import java.io.File;
24-
import java.net.URL;
2521
import java.util.ArrayList;
2622
import java.util.Arrays;
2723
import java.util.Collections;
2824
import java.util.List;
29-
import org.apache.commons.io.FileUtils;
3025
import org.apache.pinot.tools.admin.PinotAdministrator;
3126
import org.apache.pinot.tools.admin.command.QuickstartRunner;
3227

33-
import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
3428

35-
36-
public class JoinQuickStart extends QuickStartBase {
29+
public class JoinQuickStart extends Quickstart {
3730

3831
@Override
3932
public List<String> types() {
4033
return Collections.singletonList("JOIN");
4134
}
4235

43-
public void execute()
36+
@Override
37+
public void runSampleQueries(QuickstartRunner runner)
4438
throws Exception {
45-
File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
46-
47-
// Baseball stat table
48-
File baseBallStatsBaseDir = new File(quickstartTmpDir, "baseballStats");
49-
File schemaFile = new File(baseBallStatsBaseDir, "baseballStats_schema.json");
50-
File tableConfigFile = new File(baseBallStatsBaseDir, "baseballStats_offline_table_config.json");
51-
File ingestionJobSpecFile = new File(baseBallStatsBaseDir, "ingestionJobSpec.yaml");
52-
ClassLoader classLoader = Quickstart.class.getClassLoader();
53-
URL resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_schema.json");
54-
Preconditions.checkNotNull(resource);
55-
FileUtils.copyURLToFile(resource, schemaFile);
56-
resource = classLoader.getResource("examples/batch/baseballStats/baseballStats_offline_table_config.json");
57-
Preconditions.checkNotNull(resource);
58-
FileUtils.copyURLToFile(resource, tableConfigFile);
59-
resource = classLoader.getResource("examples/batch/baseballStats/ingestionJobSpec.yaml");
60-
Preconditions.checkNotNull(resource);
61-
FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
62-
QuickstartTableRequest request = new QuickstartTableRequest(baseBallStatsBaseDir.getAbsolutePath());
63-
64-
// Baseball teams dim table
65-
File dimBaseballTeamsBaseDir = new File(quickstartTmpDir, "dimBaseballTeams");
66-
schemaFile = new File(dimBaseballTeamsBaseDir, "dimBaseballTeams_schema.json");
67-
tableConfigFile = new File(dimBaseballTeamsBaseDir, "dimBaseballTeams_offline_table_config.json");
68-
ingestionJobSpecFile = new File(dimBaseballTeamsBaseDir, "ingestionJobSpec.yaml");
69-
classLoader = Quickstart.class.getClassLoader();
70-
resource = classLoader.getResource("examples/batch/dimBaseballTeams/dimBaseballTeams_schema.json");
71-
Preconditions.checkNotNull(resource);
72-
FileUtils.copyURLToFile(resource, schemaFile);
73-
resource = classLoader.getResource("examples/batch/dimBaseballTeams/dimBaseballTeams_offline_table_config.json");
74-
Preconditions.checkNotNull(resource);
75-
FileUtils.copyURLToFile(resource, tableConfigFile);
76-
resource = classLoader.getResource("examples/batch/dimBaseballTeams/ingestionJobSpec.yaml");
77-
Preconditions.checkNotNull(resource);
78-
FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
79-
QuickstartTableRequest dimTableRequest = new QuickstartTableRequest(dimBaseballTeamsBaseDir.getAbsolutePath());
80-
81-
File tempDir = new File(quickstartTmpDir, "tmp");
82-
FileUtils.forceMkdir(tempDir);
83-
QuickstartRunner runner =
84-
new QuickstartRunner(Lists.newArrayList(request, dimTableRequest), 1, 1, 3, 1, tempDir, getConfigOverrides());
85-
86-
printStatus(Quickstart.Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
87-
runner.startAll();
88-
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
89-
try {
90-
printStatus(Quickstart.Color.GREEN, "***** Shutting down offline quick start *****");
91-
runner.stop();
92-
FileUtils.deleteDirectory(quickstartTmpDir);
93-
} catch (Exception e) {
94-
e.printStackTrace();
95-
}
96-
}));
97-
printStatus(Quickstart.Color.CYAN, "***** Bootstrap baseballStats table *****");
98-
runner.bootstrapTable();
99-
100-
waitForBootstrapToComplete(null);
101-
10239
printStatus(Quickstart.Color.YELLOW, "***** Offline quickstart setup complete *****");
10340

10441
String q1 = "select count(*) from baseballStats limit 1";
@@ -120,9 +57,11 @@ public void execute()
12057
printStatus(Quickstart.Color.CYAN, "Query : " + q3);
12158
printStatus(Quickstart.Color.YELLOW, prettyPrintResponse(runner.runQuery(q3)));
12259
printStatus(Quickstart.Color.GREEN, "***************************************************");
60+
}
12361

124-
printStatus(Quickstart.Color.GREEN,
125-
"You can always go to http://localhost:9000 to play around in the query console");
62+
@Override
63+
protected int getNumQuickstartRunnerServers() {
64+
return 3;
12665
}
12766

12867
public static void main(String[] args)

‎pinot-tools/src/main/java/org/apache/pinot/tools/JsonIndexQuickStart.java‎

Lines changed: 3 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -18,81 +18,31 @@
1818
*/
1919
package org.apache.pinot.tools;
2020

21-
import com.google.common.base.Preconditions;
22-
import java.io.File;
23-
import java.net.URL;
2421
import java.util.ArrayList;
2522
import java.util.Arrays;
26-
import java.util.Collections;
2723
import java.util.List;
28-
import org.apache.commons.io.FileUtils;
29-
import org.apache.pinot.tools.Quickstart.Color;
3024
import org.apache.pinot.tools.admin.PinotAdministrator;
3125
import org.apache.pinot.tools.admin.command.QuickstartRunner;
3226

33-
import static org.apache.pinot.tools.Quickstart.prettyPrintResponse;
3427

35-
36-
public class JsonIndexQuickStart extends QuickStartBase {
28+
public class JsonIndexQuickStart extends Quickstart {
3729

3830
@Override
3931
public List<String> types() {
4032
return Arrays.asList("OFFLINE_JSON_INDEX", "OFFLINE-JSON-INDEX", "BATCH_JSON_INDEX", "BATCH-JSON-INDEX");
4133
}
4234

43-
public void execute()
35+
@Override
36+
public void runSampleQueries(QuickstartRunner runner)
4437
throws Exception {
45-
File quickstartTmpDir = new File(_dataDir, String.valueOf(System.currentTimeMillis()));
46-
File baseDir = new File(quickstartTmpDir, "githubEvents");
47-
File dataDir = new File(quickstartTmpDir, "rawdata");
48-
Preconditions.checkState(dataDir.mkdirs());
49-
50-
File schemaFile = new File(baseDir, "githubEvents_schema.json");
51-
File tableConfigFile = new File(baseDir, "githubEvents_offline_table_config.json");
52-
File ingestionJobSpecFile = new File(baseDir, "ingestionJobSpec.yaml");
53-
54-
ClassLoader classLoader = JsonIndexQuickStart.class.getClassLoader();
55-
URL resource = classLoader.getResource("examples/batch/githubEvents/githubEvents_offline_table_config.json");
56-
Preconditions.checkNotNull(resource);
57-
FileUtils.copyURLToFile(resource, tableConfigFile);
58-
resource = classLoader.getResource("examples/batch/githubEvents/githubEvents_schema.json");
59-
Preconditions.checkNotNull(resource);
60-
FileUtils.copyURLToFile(resource, schemaFile);
61-
resource = classLoader.getResource("examples/batch/githubEvents/ingestionJobSpec.yaml");
62-
Preconditions.checkNotNull(resource);
63-
FileUtils.copyURLToFile(resource, ingestionJobSpecFile);
64-
65-
QuickstartTableRequest request = new QuickstartTableRequest(baseDir.getAbsolutePath());
66-
final QuickstartRunner runner =
67-
new QuickstartRunner(Collections.singletonList(request), 1, 1, 1, 1, dataDir, getConfigOverrides());
68-
69-
printStatus(Color.CYAN, "***** Starting Zookeeper, controller, broker and server *****");
70-
runner.startAll();
71-
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
72-
try {
73-
printStatus(Color.GREEN, "***** Shutting down offline quick start *****");
74-
runner.stop();
75-
FileUtils.deleteDirectory(quickstartTmpDir);
76-
} catch (Exception e) {
77-
e.printStackTrace();
78-
}
79-
}));
80-
printStatus(Color.CYAN, "***** Bootstrap githubEvents table *****");
81-
runner.bootstrapTable();
82-
83-
waitForBootstrapToComplete(null);
84-
8538
printStatus(Color.YELLOW, "***** Offline json-index quickstart setup complete *****");
86-
8739
String q1 =
8840
"select json_extract_scalar(repo, '$.name', 'STRING'), count(*) from githubEvents where json_match(actor, "
8941
+ "'\"$.login\"=''LombiqBot''') group by 1 order by 2 desc limit 10";
9042
printStatus(Color.YELLOW, "Most contributed repos by 'LombiqBot'");
9143
printStatus(Color.CYAN, "Query : " + q1);
9244
printStatus(Color.YELLOW, prettyPrintResponse(runner.runQuery(q1)));
93-
9445
printStatus(Color.GREEN, "***************************************************");
95-
printStatus(Color.GREEN, "You can always go to http://localhost:9000 to play around in the query console");
9646
}
9747

9848
public static void main(String[] args)

0 commit comments

Comments
 (0)