Skip to content

Commit c2b36f5

Browse files
mfelgamalmfelgamal
committed
Prefixing all spark properties with livy.*
1 parent ddd99e1 commit c2b36f5

File tree

3 files changed

+20
-19
lines changed

3 files changed

+20
-19
lines changed

‎docs/interpreter/livy.md‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,15 @@ Additional requirements for the Livy interpreter are:
2323
* Livy server.
2424

2525
### Configuration
26-
We added some common configurations for spark, and you can set any configuration you want which should start with `spark.`
26+
We added some common configurations for spark, and you can set any configuration you want which should start with `livy.spark.`
2727
<table class="table-configuration">
2828
<tr>
2929
<th>Property</th>
3030
<th>Default</th>
3131
<th>Description</th>
3232
</tr>
3333
<tr>
34-
<td>spark.master</td>
34+
<td>livy.spark.master</td>
3535
<td>local[*]</td>
3636
<td>Spark master uri. ex) spark://masterhost:7077</td>
3737
</tr>
@@ -46,22 +46,22 @@ We added some common configurations for spark, and you can set any configuration
4646
<td>Max number of SparkSQL result to display.</td>
4747
</tr>
4848
<tr>
49-
<td>spark.driver.cores</td>
49+
<td>livy.spark.driver.cores</td>
5050
<td>1</td>
5151
<td>Driver cores. ex) 1, 2.</td>
5252
</tr>
5353
<tr>
54-
<td>spark.driver.memory</td>
54+
<td>livy.spark.driver.memory</td>
5555
<td>512m</td>
5656
<td>Driver memory. ex) 512m, 32g.</td>
5757
</tr>
5858
<tr>
59-
<td>spark.executor.instances</td>
59+
<td>livy.spark.executor.instances</td>
6060
<td>3</td>
6161
<td>Executor instances. ex) 1, 4.</td>
6262
</tr>
6363
<tr>
64-
<td>spark.executor.cores</td>
64+
<td>livy.spark.executor.cores</td>
6565
<td>1</td>
6666
<td>Max number of SparkSQL result to display.</td>
6767
</tr>

‎livy/src/main/java/org/apache/zeppelin/livy/LivyHelper.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,9 @@ public Integer createSession(InterpreterContext context, String kind) throws Exc
6868
Iterator<Entry<Object, Object>> it = property.entrySet().iterator();
6969
while (it.hasNext()) {
7070
Entry<Object, Object> pair = it.next();
71-
if (pair.getKey().toString().startsWith("spark.") && !pair.getValue().toString().isEmpty())
72-
conf.put(pair.getKey().toString(), pair.getValue().toString());
71+
if (pair.getKey().toString().startsWith("livy.spark.") &&
72+
!pair.getValue().toString().isEmpty())
73+
conf.put(pair.getKey().toString().substring(5), pair.getValue().toString());
7374
}
7475

7576
String confData = gson.toJson(conf);

‎livy/src/main/java/org/apache/zeppelin/livy/LivySparkInterpreter.java‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -45,21 +45,21 @@ public class LivySparkInterpreter extends Interpreter {
4545
LivySparkInterpreter.class.getName(),
4646
new InterpreterPropertyBuilder()
4747
.add("zeppelin.livy.url", DEFAULT_URL, "The URL for Livy Server.")
48-
.add("spark.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077")
49-
.add("spark.driver.cores", "", "Driver cores. ex) 1, 2")
50-
.add("spark.driver.memory", "", "Driver memory. ex) 512m, 32g")
51-
.add("spark.executor.instances", "", "Executor instances. ex) 1, 4")
52-
.add("spark.executor.cores", "", "Num cores per executor. ex) 1, 4")
53-
.add("spark.executor.memory", "",
48+
.add("livy.spark.master", LOCAL, "Spark master uri. ex) spark://masterhost:7077")
49+
.add("livy.spark.driver.cores", "", "Driver cores. ex) 1, 2")
50+
.add("livy.spark.driver.memory", "", "Driver memory. ex) 512m, 32g")
51+
.add("livy.spark.executor.instances", "", "Executor instances. ex) 1, 4")
52+
.add("livy.spark.executor.cores", "", "Num cores per executor. ex) 1, 4")
53+
.add("livy.spark.executor.memory", "",
5454
"Executor memory per worker instance. ex) 512m, 32g")
55-
.add("spark.dynamicAllocation.enabled", "", "Use dynamic resource allocation")
56-
.add("spark.dynamicAllocation.cachedExecutorIdleTimeout", "",
55+
.add("livy.spark.dynamicAllocation.enabled", "", "Use dynamic resource allocation")
56+
.add("livy.spark.dynamicAllocation.cachedExecutorIdleTimeout", "",
5757
"Remove an executor which has cached data blocks")
58-
.add("spark.dynamicAllocation.minExecutors", "",
58+
.add("livy.spark.dynamicAllocation.minExecutors", "",
5959
"Lower bound for the number of executors if dynamic allocation is enabled. ")
60-
.add("spark.dynamicAllocation.initialExecutors", "",
60+
.add("livy.spark.dynamicAllocation.initialExecutors", "",
6161
"Initial number of executors to run if dynamic allocation is enabled. ")
62-
.add("spark.dynamicAllocation.maxExecutors", "",
62+
.add("livy.spark.dynamicAllocation.maxExecutors", "",
6363
"Upper bound for the number of executors if dynamic allocation is enabled. ")
6464
.build()
6565
);

0 commit comments

Comments
 (0)