Skip to content

Commit 09c3e15

Browse files
weixiangliutang123
authored andcommitted
[MT][FEATURE] spark load in MT from 111-rc3 to 115:
change list 1.add broker plus manifest.yaml 2.[MT] spark load for mt 3.[MT] Adapt MT internal spark/yarn commands and configurations 4.[MT] add custom properties for spark load etl & del tmp hive table 5.[MT] delete spark delete spark repository and archive & improve etl job log 6.[MT] feature(sparkload): support bitmap encode features in spark load 7.[MT] feature(sparkload parquet): disable parquet dictionary 8.feature(sparkload): support bitmap binary data from hive in spark load 9.[MT] feature(sparkload): add tolas-output dependency in SparkDpp 10.fix(spark load): resolve args conflict between skip_null_value and map_side_join refactor(spark load): refactor function name each commit detail are listed in this branch: https://dev.sankuai.com/code/repo-detail/data/palo/commit/list?branch=sparkload-14-update-details or in branh 13: https://dev.sankuai.com/code/repo-detail/data/palo/commit/list?branch=refs%2Fheads%2F13 Some Spark Load changes in 0.15 to 1.1: [MT][FIX][SPARKLOAD] fix bug when partition_id exceeds integer range in spark load (apache#9073) [MT][FIX][SPARKLOAD] fix `getHashValue` of string type is always zero in spark load (apache#9135) [MT][TMP][SPARKLOAD] support `custom.global.dict.table` in spark load [MT][FEATURE][SPARKLOAD] support retry-strategy when get the spark elt job state timeout [MT][SPARKLOAD] hive table name start with tmp key word and its size should be no longer than 128 [MT][TMP][SPARKLOAD] fix min_value will be negative number when `maxGlobalDictValue` exceeds integer range (apache#9436) detail commmits' content could be found in this branch: https://dev.sankuai.com/code/repo-detail/data/palo/commit/list?branch=refs%2Fheads%2F14 [MT][TMP][FIX] fix UT in spark load [MT][FEATURE] feature(spark-dpp version): add version file for spark-dpp add spark-dpp commit id as version file when build FE [MT][SPARKLOAD] some fixes from 15 to 1.1 by wangbo36 1 not connect hive metastore when create hive table 2 avoid cast from string to bitmap expr 3 cast bytebuffer to buffer 4 handle exception in ut
1 parent 90f630e commit 09c3e15

22 files changed

Lines changed: 536 additions & 213 deletions

File tree

‎build.sh‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -631,6 +631,7 @@ fi
631631
DORIS_OUTPUT=${DORIS_OUTPUT:="${DORIS_HOME}/output/"}
632632
echo "OUTPUT DIR=${DORIS_OUTPUT}"
633633
mkdir -p "${DORIS_OUTPUT}"
634+
COMMIT_ID=$(git rev-parse --short HEAD)
634635

635636
# Copy Frontend and Backend
636637
if [[ "${BUILD_FE}" -eq 1 ]]; then
@@ -655,10 +656,12 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then
655656
mkdir -p "${DORIS_OUTPUT}/fe/conf/ssl"
656657
fi
657658

658-
if [[ "${BUILD_SPARK_DPP}" -eq 1 ]]; then
659-
install -d "${DORIS_OUTPUT}/fe/spark-dpp"
660-
rm -rf "${DORIS_OUTPUT}/fe/spark-dpp"/*
661-
cp -r -p "${DORIS_HOME}/fe/spark-dpp/target"/spark-dpp-*-jar-with-dependencies.jar "${DORIS_OUTPUT}/fe/spark-dpp"/
659+
660+
if [ ${BUILD_SPARK_DPP} -eq 1 ]; then
661+
install -d ${DORIS_OUTPUT}/fe/spark-dpp/
662+
rm -rf ${DORIS_OUTPUT}/fe/spark-dpp/*
663+
echo ${COMMIT_ID} > ${DORIS_OUTPUT}/fe/spark-dpp/version.${COMMIT_ID}
664+
cp -r -p ${DORIS_HOME}/fe/spark-dpp/target/spark-dpp-*-jar-with-dependencies.jar ${DORIS_OUTPUT}/fe/spark-dpp/
662665
fi
663666

664667
if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.doris.common;
1919

20+
import java.lang.reflect.Field;
21+
2022
public class Config extends ConfigBase {
2123

2224
@ConfField(description = {"用户自定义配置文件的路径,用于存放 fe_custom.conf。该文件中的配置会覆盖 fe.conf 中的配置",
@@ -564,7 +566,10 @@ public class Config extends ConfigBase {
564566

565567
@ConfField(mutable = true, masterOnly = true, description = {"Spark Load 所使用的 Spark 程序目录",
566568
"Spark dir for Spark Load"})
567-
public static String spark_home_default_dir = System.getenv("DORIS_HOME") + "/lib/spark2x";
569+
public static String spark_home_default_dir = "/opt/meituan/spark-2.2";
570+
571+
@ConfField(mutable = true, masterOnly = true)
572+
public static String hdfs_prefix_mt = "hdfs://dfsrouter.vip.sankuai.com:8888";
568573

569574
@ConfField(description = {"Spark load 所使用的依赖项目录", "Spark dependencies dir for Spark Load"})
570575
public static String spark_resource_path = "";
@@ -573,7 +578,7 @@ public class Config extends ConfigBase {
573578
public static String spark_launcher_log_dir = sys_log_dir + "/spark_launcher_log";
574579

575580
@ConfField(description = {"Yarn client 的路径", "Yarn client path"})
576-
public static String yarn_client_path = System.getenv("DORIS_HOME") + "/lib/yarn-client/hadoop/bin/yarn";
581+
public static String yarn_client_path = "/opt/meituan/hadoop/bin/yarn";
577582

578583
@ConfField(description = {"Yarn 配置文件的路径", "Yarn config path"})
579584
public static String yarn_config_dir = System.getenv("DORIS_HOME") + "/lib/yarn-config";
@@ -2530,6 +2535,16 @@ public class Config extends ConfigBase {
25302535
+ "and the deleted labels can be reused."
25312536
})
25322537
public static int label_num_threshold = 2000;
2538+
// 禁止在ADD PARTITION时,check quota,由于tablet非常多,check一次占用锁的时间非常长,因此需要禁用。
2539+
// 禁止在create table时check quota
2540+
@ConfField(mutable = true, masterOnly = true)
2541+
public static boolean mt_disable_check_quota_when_add_partition = true;
2542+
2543+
@ConfField(mutable = true)
2544+
public static long GET_APPID_TIMEOUT_MS = 300000L; //5min
2545+
2546+
@ConfField(mutable = true)
2547+
public static long EXEC_YARNCMD_TIMEOUT_MS = 30000L; // 30s
25332548

25342549
@ConfField(description = {"指定 internal catalog 的默认鉴权类",
25352550
"Specify the default authentication class of internal catalog"},

‎fe/fe-common/src/main/java/org/apache/doris/sparkdpp/EtlJobConfig.java‎

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import com.google.common.base.Strings;
2121
import com.google.common.collect.ImmutableMap;
2222
import com.google.common.collect.Lists;
23+
import com.google.common.collect.Maps;
2324
import com.google.gson.ExclusionStrategy;
2425
import com.google.gson.FieldAttributes;
2526
import com.google.gson.Gson;
@@ -131,12 +132,12 @@
131132
*/
132133
public class EtlJobConfig implements Serializable {
133134
// global dict
134-
public static final String GLOBAL_DICT_TABLE_NAME = "doris_global_dict_table_%d";
135-
public static final String DISTINCT_KEY_TABLE_NAME = "doris_distinct_key_table_%d_%s";
136-
public static final String DORIS_INTERMEDIATE_HIVE_TABLE_NAME = "doris_intermediate_hive_table_%d_%s";
135+
public static final String GLOBAL_DICT_TABLE_NAME = "doris_gdt_%s__%s";
136+
public static final String DISTINCT_KEY_TABLE_NAME = "tmp_doris_dkt_%s__%s__%s";
137+
public static final String DORIS_INTERMEDIATE_HIVE_TABLE_NAME = "tmp_doris_iht_%s__%s__%s";
137138

138139
// hdfsEtlPath/jobs/dbId/loadLabel/PendingTaskSignature
139-
private static final String ETL_OUTPUT_PATH_FORMAT = "%s/jobs/%d/%s/%d";
140+
private static final String ETL_OUTPUT_PATH_FORMAT = "%s/jobs/%s/%s/%d";
140141
private static final String ETL_OUTPUT_FILE_NAME_DESC_V1
141142
= "version.label.tableId.partitionId.indexId.bucket.schemaHash.parquet";
142143
// tableId.partitionId.indexId.bucket.schemaHash
@@ -145,7 +146,10 @@ public class EtlJobConfig implements Serializable {
145146

146147
// dpp result
147148
public static final String DPP_RESULT_NAME = "dpp_result.json";
148-
149+
@SerializedName(value = "dorisTableName")
150+
public String dorisTableName;
151+
@SerializedName(value = "dorisDBName")
152+
public String dorisDBName;
149153
@SerializedName(value = "tables")
150154
public Map<Long, EtlTable> tables;
151155
@SerializedName(value = "outputPath")
@@ -158,15 +162,18 @@ public class EtlJobConfig implements Serializable {
158162
public EtlJobProperty properties;
159163
@SerializedName(value = "configVersion")
160164
public ConfigVersion configVersion;
165+
@SerializedName(value="customizedProperties")
166+
public Map<String, String> customizedProperties;
161167

162168
public EtlJobConfig(Map<Long, EtlTable> tables, String outputFilePattern, String label, EtlJobProperty properties) {
163169
this.tables = tables;
164170
// set outputPath when submit etl job
165-
this.outputPath = null;
171+
this.outputPath = "";
166172
this.outputFilePattern = outputFilePattern;
167173
this.label = label;
168174
this.properties = properties;
169175
this.configVersion = ConfigVersion.V1;
176+
this.customizedProperties = Maps.newHashMap();
170177
}
171178

172179
@Override
@@ -185,8 +192,8 @@ public String getOutputPath() {
185192
return outputPath;
186193
}
187194

188-
public static String getOutputPath(String hdfsEtlPath, long dbId, String loadLabel, long taskSignature) {
189-
return String.format(ETL_OUTPUT_PATH_FORMAT, hdfsEtlPath, dbId, loadLabel, taskSignature);
195+
public static String getOutputPath(String hdfsEtlPath, String dbName, String loadLabel, long taskSignature) {
196+
return String.format(ETL_OUTPUT_PATH_FORMAT, hdfsEtlPath, dbName, loadLabel, taskSignature);
190197
}
191198

192199
public static String getOutputFilePattern(String loadLabel, FilePatternVersion filePatternVersion) {

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import org.apache.doris.analysis.FunctionName;
6565
import org.apache.doris.analysis.InstallPluginStmt;
6666
import org.apache.doris.analysis.ModifyDistributionClause;
67+
import org.apache.doris.analysis.PartitionDesc;
6768
import org.apache.doris.analysis.PartitionRenameClause;
6869
import org.apache.doris.analysis.RecoverDbStmt;
6970
import org.apache.doris.analysis.RecoverPartitionStmt;

‎fe/fe-core/src/main/java/org/apache/doris/catalog/RangePartitionInfo.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,14 @@ public static void checkPartitionColumn(Column column) throws AnalysisException
192192
}
193193
}
194194

195+
public List<Map.Entry<Long, PartitionItem>> getSortedRangeMapAll() {
196+
List<Map.Entry<Long, PartitionItem>> sortedListAll = Lists.newArrayList(idToItem.entrySet());
197+
List<Map.Entry<Long, PartitionItem>> tmpPartitionList = Lists.newArrayList(idToTempItem.entrySet());
198+
sortedListAll.addAll(tmpPartitionList);
199+
Collections.sort(sortedListAll, RangeUtils.RANGE_MAP_ENTRY_COMPARATOR);
200+
return sortedListAll;
201+
}
202+
195203
@Override
196204
public void checkPartitionItemListsMatch(List<PartitionItem> list1, List<PartitionItem> list2)
197205
throws DdlException {

‎fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java‎

Lines changed: 35 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@ public class SparkResource extends Resource {
8181
private static final String SPARK_CONFIG_PREFIX = "spark.";
8282
private static final String BROKER_PROPERTY_PREFIX = "broker.";
8383
private static final String ENV_PREFIX = "env.";
84+
private static final String CUSTOM_PROPERTY_PREFIX = "custom.";
8485
// spark uses hadoop configs in the form of spark.hadoop.*
8586
private static final String SPARK_HADOOP_CONFIG_PREFIX = "spark.hadoop.";
8687
private static final String SPARK_YARN_RESOURCE_MANAGER_ADDRESS = "spark.hadoop.yarn.resourcemanager.address";
@@ -90,6 +91,9 @@ public class SparkResource extends Resource {
9091
private static final String SPARK_YARN_RESOURCE_MANAGER_HA_RMIDS = "spark.hadoop.yarn.resourcemanager.ha.rm-ids";
9192
private static final String YARN_RESOURCE_MANAGER_ADDRESS_FOMART = "spark.hadoop.yarn.resourcemanager.address.%s";
9293
private static final String YARN_RESOURCE_MANAGER_HOSTNAME_FORMAT = "spark.hadoop.yarn.resourcemanager.hostname.%s";
94+
private static final String HADOOP_PROXY_USER = "hadoop.proxy.user";
95+
public static final String DPP_RESOURCE_DIR = "/spark-dpp/";
96+
public static final String SPARK_DPP_JAR = "spark-dpp-" + Config.spark_dpp_version + "-jar-with-dependencies.jar";
9397

9498
public enum DeployMode {
9599
CLUSTER,
@@ -116,25 +120,32 @@ public static DeployMode fromString(String deployMode) {
116120
private Map<String, String> brokerProperties;
117121
@SerializedName(value = "envConfigs")
118122
private Map<String, String> envConfigs;
123+
@SerializedName(value="customizedProperties")
124+
private Map<String, String> customizedProperties;
125+
126+
@SerializedName(value = "hadoopProxyUser")
127+
private String hadoopProxyUser;
119128

120129

121130
public SparkResource() {
122131
super();
123132
}
124133

125134
public SparkResource(String name) {
126-
this(name, Maps.newHashMap(), null, null, Maps.newHashMap(), Maps.newHashMap());
135+
this(name, Maps.newHashMap(), null, null,null, Maps.newHashMap(), Maps.newHashMap(), Maps.newHashMap());
127136
}
128137

129138
// "public" for testing
130-
public SparkResource(String name, Map<String, String> sparkConfigs, String workingDir, String broker,
131-
Map<String, String> brokerProperties, Map<String, String> envConfigs) {
139+
public SparkResource(String name, Map<String, String> sparkConfigs, String workingDir, String broker, String hadoopProxyUser,
140+
Map<String, String> brokerProperties, Map<String,String> envConfigs, Map<String, String> customizedProperties) {
132141
super(name, ResourceType.SPARK);
133142
this.sparkConfigs = sparkConfigs;
134143
this.workingDir = workingDir;
135144
this.broker = broker;
145+
this.hadoopProxyUser = hadoopProxyUser;
136146
this.brokerProperties = brokerProperties;
137147
this.envConfigs = envConfigs;
148+
this.customizedProperties = customizedProperties;
138149
}
139150

140151
public String getMaster() {
@@ -153,6 +164,10 @@ public String getBroker() {
153164
return broker;
154165
}
155166

167+
public String getHadoopProxyUser() { return hadoopProxyUser; }
168+
169+
public Map<String, String> getCustomizedProperties() { return customizedProperties; }
170+
156171
public Map<String, String> getBrokerPropertiesWithoutPrefix() {
157172
Map<String, String> properties = Maps.newHashMap();
158173
for (Map.Entry<String, String> entry : brokerProperties.entrySet()) {
@@ -190,7 +205,8 @@ public Pair<String, String> getYarnResourcemanagerAddressPair() {
190205
}
191206

192207
public SparkResource getCopiedResource() {
193-
return new SparkResource(name, Maps.newHashMap(sparkConfigs), workingDir, broker, brokerProperties, envConfigs);
208+
return new SparkResource(name, Maps.newHashMap(sparkConfigs), workingDir, broker, hadoopProxyUser,
209+
Maps.newHashMap(brokerProperties), Maps.newHashMap(envConfigs), Maps.newHashMap());
194210
}
195211

196212
@Override
@@ -268,6 +284,9 @@ private void updateProperties(Map<String, String> properties) throws DdlExceptio
268284
if (properties.containsKey(BROKER)) {
269285
broker = properties.get(BROKER);
270286
}
287+
if (properties.containsKey(HADOOP_PROXY_USER)) {
288+
hadoopProxyUser = properties.get(HADOOP_PROXY_USER);
289+
}
271290
brokerProperties.putAll(getBrokerProperties(properties));
272291
Map<String, String> env = getEnvConfig(properties);
273292
if (env.size() > 0) {
@@ -278,6 +297,7 @@ private void updateProperties(Map<String, String> properties) throws DdlExceptio
278297
}
279298
}
280299
LOG.info("updateProperties,{},{}", properties, envConfigs);
300+
customizedProperties.putAll(getCustomizedProperties(properties));
281301
}
282302

283303
@Override
@@ -328,10 +348,10 @@ protected void setProperties(Map<String, String> properties) throws DdlException
328348
+ "or not turned on ha.");
329349
}
330350
}
331-
332351
// check working dir and broker
333352
workingDir = properties.get(WORKING_DIR);
334353
broker = properties.get(BROKER);
354+
hadoopProxyUser = properties.get(HADOOP_PROXY_USER);
335355
if ((workingDir == null && broker != null) || (workingDir != null && broker == null)) {
336356
throw new DdlException("working_dir and broker should be assigned at the same time");
337357
}
@@ -382,6 +402,16 @@ private Map<String, String> getBrokerProperties(Map<String, String> properties)
382402
return brokerProperties;
383403
}
384404

405+
private Map<String, String> getCustomizedProperties(Map<String, String> properties) {
406+
Map<String, String> customizedProperties = Maps.newHashMap();
407+
for (Map.Entry<String, String> entry : properties.entrySet()) {
408+
if (entry.getKey().startsWith(CUSTOM_PROPERTY_PREFIX)) {
409+
customizedProperties.put(entry.getKey(), entry.getValue());
410+
}
411+
}
412+
return customizedProperties;
413+
}
414+
385415
@Override
386416
public void modifyProperties(Map<String, String> properties) throws DdlException {
387417
updateProperties(properties);

‎fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,7 +183,10 @@ public static CommandResult executeCommand(String cmd, String[] envp, long timeo
183183
String[] cmds = cmdList.toArray(new String[0]);
184184

185185
try {
186-
Process p = Runtime.getRuntime().exec(cmds, envp);
186+
ProcessBuilder builder = new ProcessBuilder(cmds);
187+
builder.redirectErrorStream(true);
188+
Process p = builder.start();
189+
187190
CmdWorker cmdWorker = new CmdWorker(p);
188191
cmdWorker.start();
189192

@@ -195,7 +198,7 @@ public static CommandResult executeCommand(String cmd, String[] envp, long timeo
195198
// if we get this far then we never got an exit value from the worker thread
196199
// as a result of a timeout
197200
LOG.warn("exec command [{}] timed out.", cmd);
198-
exitValue = -1;
201+
exitValue = -2;
199202
}
200203
} catch (InterruptedException ex) {
201204
cmdWorker.interrupt();

0 commit comments

Comments
 (0)