Skip to content

Commit d1f0521

Browse files
echarlesecharles
committed
Merge with master
2 parents 72ab72c + 67e0fd5 commit d1f0521

File tree

19 files changed

+262
-77
lines changed

19 files changed

+262
-77
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
spark/derby.log
1313
spark/metastore_db
1414
spark-1.*-bin-hadoop*
15+
.spark-dist
1516
zeppelin-server/derby.log
1617

1718
lens/lens-cli-hist.log

‎.travis.yml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
language: java
1717

1818
sudo: false
19+
cache:
20+
directories:
21+
- .spark-dist
1922

2023
addons:
2124
apt:
@@ -55,6 +58,7 @@ matrix:
5558
env: TEST_SELENIUM="true" SPARK_VER="1.6.0" HADOOP_VER="2.3" PROFILE="-Pspark-1.6 -Phadoop-2.3 -Ppyspark" BUILD_FLAG="package -DskipTests" TEST_FLAG="verify" TEST_PROJECTS="-pl zeppelin-interpreter,zeppelin-zengine,zeppelin-server,zeppelin-display,spark-dependencies,spark -Dtest=org.apache.zeppelin.AbstractFunctionalSuite -DfailIfNoTests=false"
5659

5760
before_install:
61+
- "ls -la .spark-dist"
5862
- "export DISPLAY=:99.0"
5963
- "sh -e /etc/init.d/xvfb start"
6064

@@ -69,7 +73,8 @@ install:
6973
before_script:
7074
- travis_retry ./testing/downloadSpark.sh $SPARK_VER $HADOOP_VER
7175
- ./testing/startSparkCluster.sh $SPARK_VER $HADOOP_VER
72-
- echo -e "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER\nexport ZEPPELIN_R_CMD=Rscript" > conf/zeppelin-env.sh
76+
- echo "export SPARK_HOME=`pwd`/spark-$SPARK_VER-bin-hadoop$HADOOP_VER" > conf/zeppelin-env.sh
77+
- tail conf/zeppelin-env.sh
7378

7479
script:
7580
- mvn $TEST_FLAG $PROFILE -B $TEST_PROJECTS

‎bin/interpreter.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,14 @@ if [[ "${INTERPRETER_ID}" == "spark" ]]; then
132132

133133
export SPARK_CLASSPATH+=":${ZEPPELIN_CLASSPATH}"
134134
fi
135+
elif [[ "${INTERPRETER_ID}" == "hbase" ]]; then
136+
if [[ -n "${HBASE_CONF_DIR}" ]]; then
137+
ZEPPELIN_CLASSPATH+=":${HBASE_CONF_DIR}"
138+
elif [[ -n "${HBASE_HOME}" ]]; then
139+
ZEPPELIN_CLASSPATH+=":${HBASE_HOME}/conf"
140+
else
141+
echo "HBASE_HOME and HBASE_CONF_DIR are not set, configuration might not be loaded"
142+
fi
135143
fi
136144

137145
addJarInDir "${LOCAL_INTERPRETER_REPO}"

‎conf/zeppelin-env.sh.template‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,9 @@
6464
# export ZEPPELIN_SPARK_MAXRESULT # Max number of SparkSQL result to display. 1000 by default.
6565
# export ZEPPELIN_WEBSOCKET_MAX_TEXT_MESSAGE_SIZE # Size in characters of the maximum text message to be received by websocket. Defaults to 1024000
6666

67+
#### HBase interpreter configuration ####
68+
69+
## To connect to HBase running on a cluster, either HBASE_HOME or HBASE_CONF_DIR must be set
70+
71+
# export HBASE_HOME= # (require) Under which HBase scripts and configuration should be
72+
# export HBASE_CONF_DIR= # (optional) Alternatively, configuration directory can be set to point to the directory that has hbase-site.xml

‎docs/interpreter/hbase.md‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,18 @@ group: manual
1010
[HBase Shell](http://hbase.apache.org/book.html#shell) is a JRuby IRB client for Apache HBase. This interpreter provides all capabilities of Apache HBase shell within Apache Zeppelin. The interpreter assumes that Apache HBase client software has been installed and it can connect to the Apache HBase cluster from the machine on where Apache Zeppelin is installed.
1111
To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/book.html#quickstart)
1212

13-
> Note: currently only HBase 1.0.x releases are supported.
13+
## HBase release supported
14+
By default, Zeppelin is built against HBase 1.0.x releases. To work with HBase 1.1.x releases, use the following build command:
15+
```bash
16+
# HBase 1.1.4
17+
mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.1.4 -Dhbase.hadoop.version=2.6.0
18+
```
19+
To work with HBase 1.2.0+, use the following build command:
20+
21+
```bash
22+
# HBase 1.2.0
23+
mvn clean package -DskipTests -Phadoop-2.6 -Dhadoop.version=2.6.0 -P build-distr -Dhbase.hbase.version=1.2.0 -Dhbase.hadoop.version=2.6.0
24+
```
1425

1526
## Configuration
1627

@@ -23,20 +34,39 @@ To get start with HBase, please see [HBase Quickstart](https://hbase.apache.org/
2334
<tr>
2435
<td>hbase.home</td>
2536
<td>/usr/lib/hbase</td>
26-
<td>Installation directory of Hbase</td>
37+
<td>Installation directory of HBase, defaults to HBASE_HOME in environment</td>
2738
</tr>
2839
<tr>
2940
<td>hbase.ruby.sources</td>
3041
<td>lib/ruby</td>
3142
<td>Path to Ruby scripts relative to 'hbase.home'</td>
3243
</tr>
3344
<tr>
34-
<td>hbase.test.mode</td>
45+
<td>zeppelin.hbase.test.mode</td>
3546
<td>false</td>
3647
<td>Disable checks for unit and manual tests</td>
3748
</tr>
3849
</table>
3950

51+
If you want to connect to HBase running on a cluster, you'll need to follow the next step.
52+
53+
### Export HBASE_HOME
54+
In **conf/zeppelin-env.sh**, export `HBASE_HOME` environment variable with your HBase installation path. This ensures `hbase-site.xml` can be loaded.
55+
56+
for example
57+
58+
```bash
59+
export HBASE_HOME=/usr/lib/hbase
60+
```
61+
62+
or, when running with CDH
63+
64+
```bash
65+
export HBASE_HOME="/opt/cloudera/parcels/CDH/lib/hbase"
66+
```
67+
68+
You can optionally export `HBASE_CONF_DIR` instead of `HBASE_HOME` should you have custom HBase configurations.
69+
4070
## Enabling the HBase Shell Interpreter
4171

4272
In a notebook, to enable the **HBase Shell** interpreter, click the **Gear** icon and select **HBase Shell**.

‎flink/src/main/java/org/apache/zeppelin/flink/FlinkInterpreter.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -249,12 +249,34 @@ public InterpreterResult interpret(String[] lines, InterpreterContext context) {
249249
Code r = null;
250250

251251
String incomplete = "";
252+
boolean inComment = false;
253+
252254
for (int l = 0; l < linesToRun.length; l++) {
253255
final String s = linesToRun[l];
254256
// check if next line starts with "." (but not ".." or "./") it is treated as an invocation
255257
if (l + 1 < linesToRun.length) {
256258
String nextLine = linesToRun[l + 1].trim();
257-
if (nextLine.startsWith(".") && !nextLine.startsWith("..") && !nextLine.startsWith("./")) {
259+
boolean continuation = false;
260+
if (nextLine.isEmpty()
261+
|| nextLine.startsWith("//") // skip empty line or comment
262+
|| nextLine.startsWith("}")
263+
|| nextLine.startsWith("object")) { // include "} object" for Scala companion object
264+
continuation = true;
265+
} else if (!inComment && nextLine.startsWith("/*")) {
266+
inComment = true;
267+
continuation = true;
268+
} else if (inComment && nextLine.lastIndexOf("*/") >= 0) {
269+
inComment = false;
270+
continuation = true;
271+
} else if (nextLine.length() > 1
272+
&& nextLine.charAt(0) == '.'
273+
&& nextLine.charAt(1) != '.' // ".."
274+
&& nextLine.charAt(1) != '/') { // "./"
275+
continuation = true;
276+
} else if (inComment) {
277+
continuation = true;
278+
}
279+
if (continuation) {
258280
incomplete += s + "\n";
259281
continue;
260282
}

‎flink/src/test/java/org/apache/zeppelin/flink/FlinkInterpreterTest.java‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,22 @@ public static void tearDown() {
4949
flink.destroy();
5050
}
5151

52+
@Test
53+
public void testNextLineInvocation() {
54+
assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n.toInt", context).code());
55+
}
56+
57+
@Test
58+
public void testNextLineComments() {
59+
assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret("\"123\"\n/*comment here\n*/.toInt", context).code());
60+
}
61+
62+
@Test
63+
public void testNextLineCompanionObject() {
64+
String code = "class Counter {\nvar value: Long = 0\n}\n // comment\n\n object Counter {\n def apply(x: Long) = new Counter()\n}";
65+
assertEquals(InterpreterResult.Code.SUCCESS, flink.interpret(code, context).code());
66+
}
67+
5268
@Test
5369
public void testSimpleStatement() {
5470
InterpreterResult result = flink.interpret("val a=1", context);
@@ -63,12 +79,6 @@ public void testSimpleStatementWithSystemOutput() {
6379
assertEquals("1", result.message());
6480
}
6581

66-
@Test
67-
public void testNextlineInvoke() {
68-
InterpreterResult result = flink.interpret("\"123\"\n .toInt", context);
69-
assertEquals("res0: Int = 123\n", result.message());
70-
}
71-
7282
@Test
7383
public void testWordCount() {
7484
flink.interpret("val text = env.fromElements(\"To be or not to be\")", context);

‎hbase/src/main/java/org/apache/zeppelin/hbase/HbaseInterpreter.java‎

Lines changed: 32 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,21 +37,20 @@
3737
import java.util.Properties;
3838

3939
/**
40-
* Support for Hbase Shell. All the commands documented here
40+
* Support for HBase Shell. All the commands documented here
4141
* http://hbase.apache.org/book.html#shell is supported.
4242
*
4343
* Requirements:
44-
* Hbase Shell should be installed on the same machine. To be more specific, the following dir.
44+
* HBase Shell should be installed on the same machine. To be more specific, the following dir.
4545
* should be available: https://github.com/apache/hbase/tree/master/hbase-shell/src/main/ruby
46-
* Hbase Shell should be able to connect to the Hbase cluster from terminal. This makes sure
46+
* HBase Shell should be able to connect to the HBase cluster from terminal. This makes sure
4747
* that the client is configured properly.
4848
*
4949
* The interpreter takes 3 config parameters:
50-
* hbase.home: Root dir. where hbase is installed. Default is /usr/lib/hbase/
50+
* hbase.home: Root directory where HBase is installed. Default is /usr/lib/hbase/
5151
* hbase.ruby.sources: Dir where shell ruby code is installed.
5252
* Path is relative to hbase.home. Default: lib/ruby
53-
* hbase.irb.load: (Testing only) Default is true.
54-
* Whether to load irb in the interpreter.
53+
* zeppelin.hbase.test.mode: (Testing only) Disable checks for unit and manual tests. Default: false
5554
*/
5655
public class HbaseInterpreter extends Interpreter {
5756
private Logger logger = LoggerFactory.getLogger(HbaseInterpreter.class);
@@ -62,11 +61,13 @@ public class HbaseInterpreter extends Interpreter {
6261
static {
6362
Interpreter.register("hbase", "hbase", HbaseInterpreter.class.getName(),
6463
new InterpreterPropertyBuilder()
65-
.add("hbase.home", "/usr/lib/hbase/", "Installation dir. of Hbase")
64+
.add("hbase.home",
65+
getSystemDefault("HBASE_HOME", "hbase.home", "/usr/lib/hbase/"),
66+
"Installation directory of HBase")
6667
.add("hbase.ruby.sources", "lib/ruby",
6768
"Path to Ruby scripts relative to 'hbase.home'")
68-
.add("hbase.test.mode", "false", "Disable checks for unit and manual tests")
69-
.build());
69+
.add("zeppelin.hbase.test.mode", "false", "Disable checks for unit and manual tests")
70+
.build());
7071
}
7172

7273
public HbaseInterpreter(Properties property) {
@@ -79,7 +80,7 @@ public void open() {
7980
this.writer = new StringWriter();
8081
scriptingContainer.setOutput(this.writer);
8182

82-
if (!Boolean.parseBoolean(getProperty("hbase.test.mode"))) {
83+
if (!Boolean.parseBoolean(getProperty("zeppelin.hbase.test.mode"))) {
8384
String hbase_home = getProperty("hbase.home");
8485
String ruby_src = getProperty("hbase.ruby.sources");
8586
Path abs_ruby_src = Paths.get(hbase_home, ruby_src).toAbsolutePath();
@@ -89,7 +90,7 @@ public void open() {
8990

9091
File f = abs_ruby_src.toFile();
9192
if (!f.exists() || !f.isDirectory()) {
92-
throw new InterpreterException("hbase ruby sources is not available at '" + abs_ruby_src
93+
throw new InterpreterException("HBase ruby sources is not available at '" + abs_ruby_src
9394
+ "'");
9495
}
9596

@@ -155,4 +156,24 @@ public List<String> completion(String buf, int cursor) {
155156
return null;
156157
}
157158

159+
private static String getSystemDefault(
160+
String envName,
161+
String propertyName,
162+
String defaultValue) {
163+
164+
if (envName != null && !envName.isEmpty()) {
165+
String envValue = System.getenv().get(envName);
166+
if (envValue != null) {
167+
return envValue;
168+
}
169+
}
170+
171+
if (propertyName != null && !propertyName.isEmpty()) {
172+
String propValue = System.getProperty(propertyName);
173+
if (propValue != null) {
174+
return propValue;
175+
}
176+
}
177+
return defaultValue;
178+
}
158179
}

‎hbase/src/test/java/org/apache/zeppelin/hbase/HbaseInterpreterTest.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ public static void setUp() throws NullPointerException {
4040
Properties properties = new Properties();
4141
properties.put("hbase.home", "");
4242
properties.put("hbase.ruby.sources", "");
43-
properties.put("hbase.test.mode", "true");
43+
properties.put("zeppelin.hbase.test.mode", "true");
4444

4545
hbaseInterpreter = new HbaseInterpreter(properties);
4646
hbaseInterpreter.open();
@@ -72,4 +72,4 @@ public void testException() {
7272
assertEquals(InterpreterResult.Code.ERROR, result.code());
7373
assertEquals("(NameError) undefined local variable or method `joke' for main:Object", result.message());
7474
}
75-
}
75+
}

‎pom.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,7 @@
241241
<groupId>org.apache.rat</groupId>
242242
<artifactId>apache-rat-plugin</artifactId>
243243
</plugin>
244+
244245
<plugin>
245246
<artifactId>maven-compiler-plugin</artifactId>
246247
<version>3.1</version>
@@ -249,6 +250,7 @@
249250
<target>1.7</target>
250251
</configuration>
251252
</plugin>
253+
252254
<!-- Test coverage plugin -->
253255
<plugin>
254256
<groupId>org.codehaus.mojo</groupId>
@@ -270,6 +272,7 @@
270272
</execution>
271273
</executions>
272274
</plugin>
275+
273276
<!-- Checkstyle plugin -->
274277
<plugin>
275278
<groupId>org.apache.maven.plugins</groupId>
@@ -488,6 +491,7 @@
488491
<exclude>conf/notebook-authorization.json</exclude>
489492
<exclude>conf/zeppelin-env.sh</exclude>
490493
<exclude>spark-*-bin*/**</exclude>
494+
<exclude>.spark-dist/**</exclude>
491495

492496
<!-- bundled from bootstrap -->
493497
<exclude>docs/assets/themes/zeppelin/bootstrap/**</exclude>
@@ -640,6 +644,13 @@
640644
</lifecycleMappingMetadata>
641645
</configuration>
642646
</plugin>
647+
648+
<plugin>
649+
<groupId>org.apache.maven.plugins</groupId>
650+
<artifactId>maven-antrun-plugin</artifactId>
651+
<version>1.7</version>
652+
</plugin>
653+
643654
</plugins>
644655
</pluginManagement>
645656
</build>

0 commit comments

Comments
 (0)