Skip to content

Commit fc59f57

Browse files
xiejiajunxiejiajun
authored
Merge pull request #4 from apache/master
sync update
2 parents 6ef9b23 + 6d5783c commit fc59f57

File tree

288 files changed

+10636
-6180
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

288 files changed

+10636
-6180
lines changed

‎.asf.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# https://cwiki.apache.org/confluence/display/INFRA/.asf.yaml+features+for+git+repositories
19+
20+
github:
21+
description: "Web-based notebook that enables data-driven, interactive data analytics and collaborative documents with SQL, Scala and more."
22+
homepage: https://zeppelin.apache.org/
23+
labels:
24+
- spark
25+
- flink
26+
- database
27+
- nosql
28+
- scala
29+
- java
30+
- big-data
31+
- zeppelin
32+
- javascript

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ Thumbs.db
9494
**/.settings/
9595
.project
9696
.settings/
97+
.factorypath
9798

9899
# intelliJ IDEA project files
99100
.idea/

‎.travis.yml‎

Lines changed: 17 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,6 @@
1515

1616
language: java
1717

18-
sudo: false
19-
20-
2118
before_cache:
2219
- sudo chown -R travis:travis $HOME/.m2
2320

@@ -53,24 +50,22 @@ env:
5350
# Interpreters does not required by zeppelin-server integration tests
5451
- INTERPRETERS='!beam,!hbase,!pig,!jdbc,!file,!flink,!ignite,!kylin,!lens,!cassandra,!elasticsearch,!bigquery,!alluxio,!scio,!livy,!groovy,!sap,!java,!geode,!neo4j,!hazelcastjet,!submarine,!sparql,!mongodb'
5552

56-
matrix:
53+
jobs:
5754
include:
5855
# Test License compliance using RAT tool
5956
- jdk: "openjdk8"
6057
dist: xenial
6158
env: SCALA_VER="2.11" PROFILE="-Prat" BUILD_FLAG="clean" TEST_FLAG="org.apache.rat:apache-rat-plugin:check" TEST_PROJECTS=""
6259

6360
# Default build command, no tests
64-
- sudo: required
65-
jdk: "openjdk8"
61+
- jdk: "openjdk8"
6662
dist: xenial
6763
env: BUILD_FLAG="clean package -T C2 -DskipTests -Pweb-angular" TEST_FLAG="test -DskipTests -Pweb-angular"
6864

6965
# Run e2e tests (in zeppelin-web)
7066
# chrome dropped the support for precise (ubuntu 12.04), so need to use trusty
7167
# also, can't use JDK 7 in trusty: https://github.com/travis-ci/travis-ci/issues/7884
7268
- os: linux
73-
sudo: false
7469
dist: xenial
7570
jdk: "openjdk8"
7671
env: CI="true" WEB_E2E="true" PYTHON="2" SCALA_VER="2.11" SPARK_VER="2.1.0" HADOOP_VER="2.6" PROFILE="-Phadoop2 -Pscala-2.11" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web" TEST_PROJECTS="-Pweb-e2e"
@@ -81,7 +76,6 @@ matrix:
8176

8277
# Run tests (in zeppelin-web-angular)
8378
- os: linux
84-
sudo: false
8579
dist: xenial
8680
jdk: "openjdk8"
8781
env: CI="true" BUILD_FLAG="clean -DskipTests -DskipRat" TEST_FLAG="package -DskipRat" MODULES="-pl ${INTERPRETERS}" TEST_MODULES="-pl zeppelin-web-angular -Pweb-angular"
@@ -94,8 +88,7 @@ matrix:
9488
# Several tests were excluded from this configuration due to the following issues:
9589
# HeliumApplicationFactoryTest - https://issues.apache.org/jira/browse/ZEPPELIN-2470
9690
# After issues are fixed these tests need to be included back by removing them from the "-Dtests.to.exclude" property
97-
- sudo: required
98-
jdk: "openjdk8"
91+
- jdk: "openjdk8"
9992
dist: xenial
10093
env: BUILD_PLUGINS="true" PYTHON="3" R="true" PROFILE="-Phelium-dev -Pexamples" BUILD_FLAG="install -Pbuild-distr -DskipRat -DskipTests" TEST_FLAG="verify -Pusing-packaged-distr -DskipRat" MODULES="-pl zeppelin-server,zeppelin-web,spark/spark-dependencies,markdown,angular,shell -am" TEST_PROJECTS="-Dtests.to.exclude=**/org/apache/zeppelin/spark/*,**/HeliumApplicationFactoryTest.java -DfailIfNoTests=false"
10194

@@ -114,68 +107,57 @@ matrix:
114107
# Run Spark integration test and unit test
115108

116109
# Run spark integration of in one zeppelin instance (2.4, 2.3, 2.2)
117-
- sudo: required
118-
jdk: "openjdk8"
110+
- jdk: "openjdk8"
119111
dist: xenial
120112
env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.11" PROFILE="-Phadoop2 -Pintegration" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter-integration,zeppelin-web,spark/spark-dependencies,markdown -am" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest24,SparkIntegrationTest24,ZeppelinSparkClusterTest23,SparkIntegrationTest23,ZeppelinSparkClusterTest22,SparkIntegrationTest22 -DfailIfNoTests=false"
121113

122114
# Run spark integration of in one zeppelin instance (2.1, 2.0, 1.6)
123-
- sudo: required
124-
jdk: "openjdk8"
115+
- jdk: "openjdk8"
125116
dist: xenial
126117
env: BUILD_PLUGINS="true" PYTHON="2" SCALA_VER="2.11" PROFILE="-Phadoop2 -Pintegration" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter-integration,zeppelin-web,spark/spark-dependencies,markdown -am" TEST_PROJECTS="-Dtest=ZeppelinSparkClusterTest21,SparkIntegrationTest21,ZeppelinSparkClusterTest20,SparkIntegrationTest20,ZeppelinSparkClusterTest16,SparkIntegrationTest16 -DfailIfNoTests=false"
127118

128119
# JdbcIntegrationTest, Unit test of Spark 2.4 (Scala-2.11)
129-
- sudo: required
130-
jdk: "openjdk8"
120+
- jdk: "openjdk8"
131121
dist: xenial
132122
env: BUILD_PLUGINS="true" PYTHON="3" SCALA_VER="2.11" PROFILE="-Pspark-2.4 -Pspark-scala-2.11 -Phadoop2 -Pintegration" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl zeppelin-interpreter-integration,jdbc,zeppelin-web,spark/spark-dependencies,markdown -am" TEST_PROJECTS="-Dtest=JdbcIntegrationTest,org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false"
133123

134124
# Unit test of Spark 2.4 (Scala-2.12)
135-
- sudo: required
136-
jdk: "openjdk8"
125+
- jdk: "openjdk8"
137126
dist: xenial
138127
env: BUILD_PLUGINS="false" PYTHON="3" SCALA_VER="2.12" PROFILE="-Pspark-2.4 -Pspark-scala-2.12 -Phadoop2" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl spark/spark-dependencies -am" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,org.apache.zeppelin.kotlin.* -DfailIfNoTests=false"
139128

140129
# Unit test of Spark 2.3 (Scala-2.11) and Unit test python, jupyter and r interpreter under python2
141-
- sudo: required
142-
jdk: "openjdk8"
130+
- jdk: "openjdk8"
143131
dist: xenial
144132
env: BUILD_PLUGINS="false" PYTHON="3" SCALA_VER="2.11" PROFILE="-Pspark-2.3 -Pspark-scala-2.11 -Phadoop2" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl spark/spark-dependencies -am" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false"
145133

146134
# Unit test of Spark 2.2 (Scala-2.10) and Unit test python, jupyter and r interpreter under python3
147-
- sudo: required
148-
jdk: "openjdk8"
135+
- jdk: "openjdk8"
149136
dist: xenial
150137
env: BUILD_PLUGINS="false" PYTHON="3" SCALA_VER="2.10" PROFILE="-Pspark-2.2 -Pspark-scala-2.10 -Phadoop2" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl spark/spark-dependencies -am" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.*,apache.zeppelin.python.*,apache.zeppelin.jupyter.*,apache.zeppelin.r.* -DfailIfNoTests=false"
151138

152139
# Unit test of Spark 2.1 (Scala-2.10)
153-
- sudo: required
154-
jdk: "openjdk8"
140+
- jdk: "openjdk8"
155141
dist: xenial
156142
env: BUILD_PLUGINS="false" PYTHON="3" SCALA_VER="2.10" PROFILE="-Pspark-2.1 -Phadoop2 -Pspark-scala-2.10 -Pintegration" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl spark/spark-dependencies -am" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
157143

158144
# Unit test of Spark 2.0 (Scala-2.10), Use python 3.5 because spark 2.0 doesn't support python 3.6 +
159-
- sudo: required
160-
jdk: "openjdk8"
145+
- jdk: "openjdk8"
161146
dist: xenial
162147
env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.10" PROFILE="-Pspark-2.0 -Phadoop2 -Pspark-scala-2.10" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl spark/spark-dependencies -am" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
163148

164149
# Unit test of Spark 1.6 (Scala-2.10)
165-
- sudo: required
166-
jdk: "openjdk8"
150+
- jdk: "openjdk8"
167151
dist: xenial
168152
env: BUILD_PLUGINS="false" PYTHON="2" SCALA_VER="2.10" PROFILE="-Pspark-1.6 -Phadoop2 -Pspark-scala-2.10" R="true" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="test -DskipRat" MODULES="-pl spark/spark-dependencies -am" TEST_PROJECTS="-Dtest=org.apache.zeppelin.spark.* -DfailIfNoTests=false"
169153

170154
# Test python/pyspark with python 2, livy 0.5
171-
- sudo: required
172-
dist: xenial
155+
- dist: xenial
173156
jdk: "openjdk8"
174157
env: PYTHON="2" SCALA_VER="2.10" SPARK_VER="1.6.3" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="-Pspark-1.6 -Phadoop2 -Pscala-2.10" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy -am" TEST_PROJECTS=""
175158

176159
# Test livy 0.5 with spark 2.2.0 under python3
177-
- sudo: required
178-
dist: xenial
160+
- dist: xenial
179161
jdk: "openjdk8"
180162
env: PYTHON="3" SPARK_VER="2.2.0" HADOOP_VER="2.6" LIVY_VER="0.5.0-incubating" PROFILE="" BUILD_FLAG="install -DskipTests -DskipRat" TEST_FLAG="verify -DskipRat" MODULES="-pl livy -am" TEST_PROJECTS=""
181163

@@ -188,7 +170,7 @@ before_install:
188170
- gitlog=$(git log $TRAVIS_COMMIT_RANGE 2>/dev/null) || gitlog=""
189171
- clearcache=$(echo $gitlog | grep -c -E "clear bower|bower clear" || true)
190172
- if [ "$hasbowerchanged" -gt 0 ] || [ "$clearcache" -gt 0 ]; then echo "Clearing bower_components cache"; rm -r zeppelin-web/bower_components; npm cache verify; else echo "Using cached bower_components."; fi
191-
- echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
173+
- echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=warn'" >> ~/.mavenrc
192174
- bash -x ./testing/install_external_dependencies.sh
193175
- ls -la .spark-dist ${HOME}/.m2/repository/.cache/maven-download-plugin || true
194176
- ls .node_modules && cp -r .node_modules zeppelin-web/node_modules || echo "node_modules are not cached"
@@ -215,7 +197,7 @@ before_script:
215197
# https://docs.travis-ci.com/user/gui-and-headless-browsers/#Using-xvfb-to-Run-Tests-That-Require-a-GUI
216198
- if [[ -n $TEST_MODULES ]]; then export DISPLAY=:99.0; sh -e /etc/init.d/xvfb start; sleep 3; fi
217199
# display info log for debugging
218-
- if [[ -n $TEST_MODULES ]]; then echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxPermSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=info'" > ~/.mavenrc; fi
200+
- if [[ -n $TEST_MODULES ]]; then echo "MAVEN_OPTS='-Xms1024M -Xmx2048M -XX:MaxMetaspaceSize=1024m -XX:-UseGCOverheadLimit -Dorg.slf4j.simpleLogger.defaultLogLevel=info'" > ~/.mavenrc; fi
219201

220202
script:
221203
- if [[ -n $TEST_MODULES ]]; then export MODULES="${TEST_MODULES}"; fi
@@ -239,3 +221,4 @@ after_failure:
239221
- cat livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*/*/*/stdout
240222
- cat livy/target/tmp/livy-int-test/MiniYarnMain/target/org.apache.livy.test.framework.MiniYarnMain/*/*/*/stderr
241223
- cat zeppelin-zengine/target/org.apache.zeppelin.interpreter.MiniHadoopCluster/*/*/*/stdout
224+
- cat flink/*.log

‎beam/pom.xml‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@
4444
<!-- library versions -->
4545
<netty.version>4.1.42.Final</netty.version>
4646
<servlet.api.version>3.1.0</servlet.api.version>
47-
<commons.exec.version>1.3</commons.exec.version>
4847
<avro.version>1.8.1</avro.version> <!-- should match beam dependency -->
4948
</properties>
5049

‎bigquery/pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
<!-- library versions -->
4343
<bigquery.api.version>v2-rev20190917-1.30.3</bigquery.api.version>
4444
<gson.version>2.8.6</gson.version>
45-
<guava.version>22.0</guava.version>
45+
<guava.version>24.1.1-jre</guava.version>
4646

4747
<interpreter.name>bigquery</interpreter.name>
4848
</properties>

‎bin/common.cmd‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,11 @@ if not defined ZEPPELIN_ENCODING (
5858
)
5959

6060
if not defined ZEPPELIN_MEM (
61-
set ZEPPELIN_MEM=-Xms1024m -Xmx1024m -XX:MaxPermSize=512m
61+
set ZEPPELIN_MEM=-Xms1024m -Xmx1024m
6262
)
6363

6464
if not defined ZEPPELIN_INTP_MEM (
65-
set ZEPPELIN_INTP_MEM=-Xms1024m -Xmx1024m -XX:MaxPermSize=512m
65+
set ZEPPELIN_INTP_MEM=-Xms1024m -Xmx1024m
6666
)
6767

6868
if not defined ZEPPELIN_JAVA_OPTS (

‎bin/common.sh‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,20 @@ fi
6262

6363
ZEPPELIN_CLASSPATH+=":${ZEPPELIN_CONF_DIR}"
6464

65+
function check_java_version() {
66+
java_ver_output=$("${JAVA:-java}" -version 2>&1)
67+
jvmver=$(echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR==1 {print $2}' | cut -d\- -f1)
68+
JVM_VERSION=$(echo "$jvmver"|sed -e 's|^\([0-9][0-9]*\)\..*$|\1|')
69+
if [ "$JVM_VERSION" = "1" ]; then
70+
JVM_VERSION=$(echo "$jvmver"|sed -e 's|^1\.\([0-9][0-9]*\)\..*$|\1|')
71+
fi
72+
73+
if [ "$JVM_VERSION" -lt 8 ] || ([ "$JVM_VERSION" -eq 8 ] && [ "${jvmver#*_}" -lt 151 ]) ; then
74+
echo "Apache Zeppelin requires either Java 8 update 151 or newer"
75+
exit 1;
76+
fi
77+
}
78+
6579
function addEachJarInDir(){
6680
if [[ -d "${1}" ]]; then
6781
for jar in $(find -L "${1}" -maxdepth 1 -name '*jar'); do
@@ -118,11 +132,11 @@ if [[ -z "${ZEPPELIN_ENCODING}" ]]; then
118132
fi
119133

120134
if [[ -z "${ZEPPELIN_MEM}" ]]; then
121-
export ZEPPELIN_MEM="-Xms1024m -Xmx1024m -XX:MaxPermSize=512m"
135+
export ZEPPELIN_MEM="-Xms1024m -Xmx1024m"
122136
fi
123137

124138
if [[ -z "${ZEPPELIN_INTP_MEM}" ]]; then
125-
export ZEPPELIN_INTP_MEM="-Xms1024m -Xmx2048m -XX:MaxPermSize=512m"
139+
export ZEPPELIN_INTP_MEM="-Xms1024m -Xmx2048m"
126140
fi
127141

128142
JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING} ${ZEPPELIN_MEM}"

‎bin/interpreter.sh‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,36 @@
1717
#
1818

1919

20-
bin=$(dirname "${BASH_SOURCE-$0}")
21-
bin=$(cd "${bin}">/dev/null; pwd)
20+
bin="$(dirname "${BASH_SOURCE-$0}")"
21+
bin="$(cd "${bin}">/dev/null; pwd)"
2222

2323
function usage() {
2424
echo "usage) $0 -p <port> -r <intp_port> -d <interpreter dir to load> -l <local interpreter repo dir to load> -g <interpreter group name>"
2525
}
2626

27+
# pre-requisites for checking that we're running in container
28+
if [ -f /proc/self/cgroup ] && [ -n "$(command -v getent)" ]; then
29+
# checks if we're running in container...
30+
if awk -F: '/cpu/ && $3 ~ /^\/$/{ c=1 } END { exit c }' /proc/self/cgroup; then
31+
# Check whether there is a passwd entry for the container UID
32+
myuid="$(id -u)"
33+
mygid="$(id -g)"
34+
# turn off -e for getent because it will return error code in anonymous uid case
35+
set +e
36+
uidentry="$(getent passwd "$myuid")"
37+
set -e
38+
39+
# If there is no passwd entry for the container UID, attempt to create one
40+
if [ -z "$uidentry" ] ; then
41+
if [ -w /etc/passwd ] ; then
42+
echo "zeppelin:x:$myuid:$mygid:anonymous uid:$Z_HOME:/bin/false" >> /etc/passwd
43+
else
44+
echo "Container ENTRYPOINT failed to add passwd entry for anonymous UID"
45+
fi
46+
fi
47+
fi
48+
fi
49+
2750
while getopts "hc:p:r:i:d:l:v:u:g:" o; do
2851
case ${o} in
2952
h)
@@ -69,6 +92,9 @@ fi
6992

7093
. "${bin}/common.sh"
7194

95+
check_java_version
96+
97+
7298
ZEPPELIN_INTERPRETER_API_JAR=$(find "${ZEPPELIN_HOME}/interpreter" -name 'zeppelin-interpreter-shaded-*.jar')
7399
ZEPPELIN_INTP_CLASSPATH="${CLASSPATH}:${ZEPPELIN_INTERPRETER_API_JAR}"
74100

@@ -210,6 +236,10 @@ elif [[ "${INTERPRETER_ID}" == "flink" ]]; then
210236

211237
if [[ -n "${HADOOP_CONF_DIR}" ]] && [[ -d "${HADOOP_CONF_DIR}" ]]; then
212238
ZEPPELIN_INTP_CLASSPATH+=":${HADOOP_CONF_DIR}"
239+
if ! [ -x "$(command -v hadoop)" ]; then
240+
echo 'Error: hadoop is not in PATH when HADOOP_CONF_DIR is specified.'
241+
exit 1
242+
fi
213243
ZEPPELIN_INTP_CLASSPATH+=":`hadoop classpath`"
214244
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR}
215245
else

0 commit comments

Comments
 (0)