Skip to content

Commit 26fc59b

Browse files
mfelgamalmfelgamal
committed
Refactoring of the code
1 parent 3a2bd85 commit 26fc59b

File tree

6 files changed

+50
-58
lines changed

6 files changed

+50
-58
lines changed

‎beam/pom.xml‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55

66

7+
78
<parent>
89
<artifactId>zeppelin</artifactId>
910
<groupId>org.apache.zeppelin</groupId>
@@ -15,22 +16,26 @@
1516
<artifactId>zeppelin-beam</artifactId>
1617
<version>0.7.0-SNAPSHOT</version>
1718

18-
19-
19+
<repositories>
20+
<repository>
21+
<id>apache-beam</id>
22+
<url>https://repository.apache.org/content/repositories/snapshots/org/apache/beam/</url>
23+
</repository>
24+
</repositories>
2025

2126
<dependencies>
2227
<!-- <dependency> <groupId>com.google.cloud.dataflow</groupId> <artifactId>google-cloud-dataflow-java-sdk-all</artifactId>
2328
<version>1.6.0</version> </dependency> <dependency> <groupId>org.apache.beam</groupId>
24-
<artifactId>beam-runners-parent</artifactId> <version>0.2.0-incubating-SNAPSHOT</version>
29+
<artifactId>beam-runners-parent</artifactId> <version>0.1.0-incubating</version>
2530
<type>pom</type> </dependency> <dependency> <groupId>org.apache.beam</groupId>
26-
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId> <version>0.2.0-incubating-SNAPSHOT</version>
31+
<artifactId>beam-sdks-java-io-google-cloud-platform</artifactId> <version>0.1.0-incubating</version>
2732
</dependency> <dependency> <groupId>org.apache.beam</groupId> <artifactId>beam-parent</artifactId>
28-
<version>0.2.0-incubating-SNAPSHOT</version> <type>pom</type> </dependency>
33+
<version>0.1.0-incubating</version> <type>pom</type> </dependency>
2934
<dependency> <groupId>org.apache.beam</groupId> <artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
3035
<version>0.2.0-incubating</version> </dependency> <dependency> <groupId>org.apache.beam</groupId>
3136
<artifactId>beam-runners-core-java</artifactId> <version>0.2.0-incubating</version>
3237
</dependency> <dependency> <groupId>org.apache.beam</groupId> <artifactId>beam-sdks-java-core</artifactId>
33-
<version>0.2.0-incubating-SNAPSHOT</version> </dependency> -->
38+
<version>0.1.0-incubating</version> </dependency> -->
3439

3540
<!--
3641
<dependency>
@@ -187,14 +192,14 @@
187192
<dependency>
188193
<groupId>org.apache.beam</groupId>
189194
<artifactId>beam-runners-parent</artifactId>
190-
<version>0.2.0-incubating-SNAPSHOT</version>
195+
<version>0.1.0-incubating</version>
191196
<type>pom</type>
192197
</dependency>
193198
<!--
194199
<dependency>
195200
<groupId>org.apache.beam</groupId>
196201
<artifactId>beam-examples-java</artifactId>
197-
<version>0.2.0-incubating-SNAPSHOT</version>
202+
<version>0.1.0-incubating</version>
198203
199204
<exclusions>
200205
<exclusion>
@@ -207,7 +212,7 @@
207212
<dependency>
208213
<groupId>org.apache.beam</groupId>
209214
<artifactId>beam-runners-core-java</artifactId>
210-
<version>0.2.0-incubating-SNAPSHOT</version>
215+
<version>0.1.0-incubating</version>
211216
<exclusions>
212217
<exclusion>
213218
<artifactId>google-http-client-jackson2</artifactId>
@@ -218,13 +223,13 @@
218223
<dependency>
219224
<groupId>org.apache.beam</groupId>
220225
<artifactId>beam-runners-direct-java</artifactId>
221-
<version>0.2.0-incubating-SNAPSHOT</version>
226+
<version>0.1.0-incubating</version>
222227

223228
</dependency>
224229
<dependency>
225230
<groupId>org.apache.beam</groupId>
226231
<artifactId>beam-runners-flink_2.10</artifactId>
227-
<version>0.2.0-incubating-SNAPSHOT</version>
232+
<version>0.1.0-incubating</version>
228233

229234
<exclusions>
230235
<exclusion>
@@ -241,7 +246,7 @@
241246
<dependency>
242247
<groupId>org.apache.beam</groupId>
243248
<artifactId>beam-runners-flink_2.10-examples</artifactId>
244-
<version>0.2.0-incubating-SNAPSHOT</version>
249+
<version>0.1.0-incubating</version>
245250

246251
<exclusions>
247252
<exclusion>
@@ -254,7 +259,7 @@
254259
<dependency>
255260
<groupId>org.apache.beam</groupId>
256261
<artifactId>beam-runners-google-cloud-dataflow-java</artifactId>
257-
<version>0.2.0-incubating-SNAPSHOT</version>
262+
<version>0.1.0-incubating</version>
258263
259264
<exclusions>
260265
<exclusion>
@@ -267,7 +272,7 @@
267272

268273
<groupId>org.apache.beam</groupId>
269274
<artifactId>beam-runners-spark</artifactId>
270-
<version>0.2.0-incubating-SNAPSHOT</version>
275+
<version>0.1.0-incubating</version>
271276
<type>jar</type>
272277
</dependency>
273278

@@ -289,14 +294,13 @@
289294
290295
-->
291296

292-
<!--
297+
293298
<dependency>
294299
<groupId>${project.groupId}</groupId>
295300
<artifactId>zeppelin-interpreter</artifactId>
296301
<version>${project.version}</version>
297302
<scope>provided</scope>
298303
</dependency>
299-
-->
300304
<dependency>
301305
<groupId>org.apache.commons</groupId>
302306
<artifactId>commons-exec</artifactId>

‎beam/src/main/java/org/apache/zeppelin/beam/BeamInterpreter.java‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package org.apache.zeppelin.beam;
22

3-
43
import java.io.File;
54
import java.io.PrintWriter;
65
import java.io.StringWriter;
@@ -23,35 +22,33 @@
2322
*/
2423
public class BeamInterpreter extends Interpreter {
2524

26-
private String host = "http://localhost:8001";
27-
private InterpreterContext context;
28-
2925
public BeamInterpreter(Properties property) {
3026
super(property);
3127
}
3228

33-
34-
public static void main(String[] args) {
35-
36-
}
37-
3829
@Override
3930
public void open() {
4031

4132
}
4233

4334
@Override
4435
public void close() {
45-
36+
File dir = new File(".");
37+
for (int i = 0; i < dir.list().length; i++) {
38+
File f = dir.listFiles()[i];
39+
System.out.println(f.getAbsolutePath());
40+
if (f.getAbsolutePath().contains(".class"))
41+
f.delete();
42+
}
4643
}
4744

4845
@Override
4946
public InterpreterResult interpret(String st, InterpreterContext context) {
5047

51-
String uuid = "C" + UUID.randomUUID().toString().replace("-", "");
48+
String className = "C" + UUID.randomUUID().toString().replace("-", "");
5249

5350
try {
54-
String msg = CompileSourceInMemory.execute(uuid, st);
51+
String msg = CompileSourceInMemory.execute(className, st);
5552
return new InterpreterResult(InterpreterResult.Code.SUCCESS, msg);
5653
} catch (Exception e) {
5754
e.printStackTrace();

‎beam/src/main/java/org/apache/zeppelin/beam/CompileSourceInMemory.java‎

Lines changed: 15 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,17 @@
1414
import com.thoughtworks.qdox.model.JavaSource;
1515

1616
import java.io.ByteArrayOutputStream;
17-
import java.io.File;
1817
import java.io.PrintStream;
1918
import java.io.PrintWriter;
2019
import java.io.StringReader;
2120
import java.io.StringWriter;
2221
import java.lang.reflect.InvocationTargetException;
2322
import java.net.URI;
24-
import java.net.URL;
25-
import java.net.URLClassLoader;
2623
import java.util.Arrays;
2724
import java.util.List;
2825

2926
/**
30-
* @author admin
27+
* @author Mahmoud
3128
*
3229
*/
3330
public class CompileSourceInMemory {
@@ -39,43 +36,37 @@ public static String execute(String className, String code) throws Exception {
3936
JavaProjectBuilder builder = new JavaProjectBuilder();
4037
JavaSource src = builder.addSource(new StringReader(code));
4138

42-
// List<String> imports = src.getImports();
43-
// String importsString = "";
44-
//
45-
// for (int i = 0; i < imports.size(); i++) {
46-
// importsString += "import " + imports.get(i) + ";\n";
47-
// }
48-
4939
List<JavaClass> classes = src.getClasses();
50-
String classesSt = "";
51-
String classMain = "", classMainName = "";
40+
String classMainName = null;
5241
for (int i = 0; i < classes.size(); i++) {
5342
boolean hasMain = false;
5443
for (int j = 0; j < classes.get(i).getMethods().size(); j++) {
5544
if (classes.get(i).getMethods().get(j).getName().equals("main")) {
45+
classMainName = classes.get(i).getName();
5646
hasMain = true;
5747
break;
5848
}
5949
}
60-
if (hasMain == true) {
61-
classMain = classes.get(i).getCodeBlock() + "\n";
62-
classMainName = classes.get(i).getName();
63-
} else
64-
classesSt += classes.get(i).getCodeBlock() + "\n";
50+
if (hasMain == true)
51+
break;
6552

6653
}
54+
55+
if (classMainName == null)
56+
throw new Exception("There isn't any class containing Main method.");
57+
6758
code = code.replace(classMainName, className);
6859

6960
StringWriter writer = new StringWriter();
7061
PrintWriter out = new PrintWriter(writer);
7162

7263
out.println(code);
7364
out.close();
65+
7466

75-
System.out.println(writer.toString());
7667

7768
JavaFileObject file = new JavaSourceFromString(className, writer.toString());
78-
69+
7970
Iterable<? extends JavaFileObject> compilationUnits = Arrays.asList(file);
8071

8172
ByteArrayOutputStream baosOut = new ByteArrayOutputStream();
@@ -100,20 +91,18 @@ public static String execute(String className, String code) throws Exception {
10091
}
10192
if (success) {
10293
try {
103-
URLClassLoader classLoader = URLClassLoader.newInstance(new URL[] { new File("").toURI()
104-
.toURL() });
105-
Class.forName(className, true, classLoader)
106-
.getDeclaredMethod("main", new Class[] { String[].class })
107-
.invoke(null, new Object[] { null });
94+
95+
Class.forName(className).getDeclaredMethod("main", new Class[] { String[].class })
96+
.invoke(null, new Object[] { null });
10897

10998
System.out.flush();
11099
System.err.flush();
111100

112101
System.setOut(oldOut);
113102
System.setErr(oldErr);
114103

115-
classLoader.clearAssertionStatus();
116104

105+
117106
return baosOut.toString();
118107
} catch (ClassNotFoundException e) {
119108
e.printStackTrace(newErr);

‎conf/interpreter-list‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ cassandra org.apache.zeppelin:zeppelin-cassandra_2.11:0.6.1 Cassandr
2424
elasticsearch org.apache.zeppelin:zeppelin-elasticsearch:0.6.1 Elasticsearch interpreter
2525
file org.apache.zeppelin:zeppelin-file:0.6.1 HDFS file interpreter
2626
flink org.apache.zeppelin:zeppelin-flink_2.11:0.6.1 Flink interpreter built with Scala 2.11
27+
beam org.apache.zeppelin:zeppelin-beam:0.6.1 Beam interpreter
2728
hbase org.apache.zeppelin:zeppelin-hbase:0.6.1 Hbase interpreter
2829
ignite org.apache.zeppelin:zeppelin-ignite_2.11:0.6.1 Ignite interpreter built with Scala 2.11
2930
jdbc org.apache.zeppelin:zeppelin-jdbc:0.6.1 Jdbc interpreter

‎conf/zeppelin-site.xml.template‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,13 +184,13 @@
184184

185185
<property>
186186
<name>zeppelin.interpreters</name>
187-
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter</value>
187+
<value>org.apache.zeppelin.spark.SparkInterpreter,org.apache.zeppelin.spark.PySparkInterpreter,org.apache.zeppelin.rinterpreter.RRepl,org.apache.zeppelin.rinterpreter.KnitR,org.apache.zeppelin.spark.SparkRInterpreter,org.apache.zeppelin.spark.SparkSqlInterpreter,org.apache.zeppelin.spark.DepInterpreter,org.apache.zeppelin.markdown.Markdown,org.apache.zeppelin.angular.AngularInterpreter,org.apache.zeppelin.shell.ShellInterpreter,org.apache.zeppelin.file.HDFSFileInterpreter,org.apache.zeppelin.flink.FlinkInterpreter,,org.apache.zeppelin.python.PythonInterpreter,org.apache.zeppelin.lens.LensInterpreter,org.apache.zeppelin.ignite.IgniteInterpreter,org.apache.zeppelin.ignite.IgniteSqlInterpreter,org.apache.zeppelin.cassandra.CassandraInterpreter,org.apache.zeppelin.geode.GeodeOqlInterpreter,org.apache.zeppelin.postgresql.PostgreSqlInterpreter,org.apache.zeppelin.jdbc.JDBCInterpreter,org.apache.zeppelin.kylin.KylinInterpreter,org.apache.zeppelin.elasticsearch.ElasticsearchInterpreter,org.apache.zeppelin.scalding.ScaldingInterpreter,org.apache.zeppelin.alluxio.AlluxioInterpreter,org.apache.zeppelin.hbase.HbaseInterpreter,org.apache.zeppelin.livy.LivySparkInterpreter,org.apache.zeppelin.livy.LivyPySparkInterpreter,org.apache.zeppelin.livy.LivySparkRInterpreter,org.apache.zeppelin.livy.LivySparkSQLInterpreter,org.apache.zeppelin.bigquery.BigQueryInterpreter,org.apache.zeppelin.beam.BeamInterpreter</value>
188188
<description>Comma separated interpreter configurations. First interpreter become a default</description>
189189
</property>
190190

191191
<property>
192192
<name>zeppelin.interpreter.group.order</name>
193-
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery</value>
193+
<value>spark,md,angular,sh,livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,scalding,jdbc,hbase,bigquery,beam</value>
194194
<description></description>
195195
</property>
196196

‎zeppelin-zengine/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -529,15 +529,16 @@ public static enum ConfVars {
529529
+ "org.apache.zeppelin.scalding.ScaldingInterpreter,"
530530
+ "org.apache.zeppelin.jdbc.JDBCInterpreter,"
531531
+ "org.apache.zeppelin.hbase.HbaseInterpreter,"
532-
+ "org.apache.zeppelin.bigquery.BigQueryInterpreter"),
532+
+ "org.apache.zeppelin.bigquery.BigQueryInterpreter",
533+
+ "org.apache.zeppelin.beam.BeamInterpreter"),
533534
ZEPPELIN_INTERPRETER_JSON("zeppelin.interpreter.setting", "interpreter-setting.json"),
534535
ZEPPELIN_INTERPRETER_DIR("zeppelin.interpreter.dir", "interpreter"),
535536
ZEPPELIN_INTERPRETER_LOCALREPO("zeppelin.interpreter.localRepo", "local-repo"),
536537
ZEPPELIN_INTERPRETER_CONNECT_TIMEOUT("zeppelin.interpreter.connect.timeout", 30000),
537538
ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize", 10),
538539
ZEPPELIN_INTERPRETER_GROUP_ORDER("zeppelin.interpreter.group.order", "spark,md,angular,sh,"
539540
+ "livy,alluxio,file,psql,flink,python,ignite,lens,cassandra,geode,kylin,elasticsearch,"
540-
+ "scalding,jdbc,hbase,bigquery"),
541+
+ "scalding,jdbc,hbase,bigquery,beam"),
541542
ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
542543
ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
543544
// use specified notebook (id) as homescreen

0 commit comments

Comments
 (0)