Skip to content

Commit 2a680f5

Browse files
authored
[Fix](pythonUdf) Fix pythonUdf helper cmd coredump when enable_python_udf_support is not set (#61196)
When `enable_python_udf_support` is not set to true, a null pointer dereference occurs when invoking helper commands (`SHOW PYTHON VERSION/ PACKAGES IN`). ```text /home/zcp/repo_center/doris_master/doris/be/src/udf/python/python_env.cpp:305:38: runtime error: member call on null pointer of type 'doris::PythonEnvScanner' #0 0x559edede162c in doris::PythonVersionManager::env_infos_to_thrift() const /home/zcp/repo_center/doris_master/doris/be/src/udf/python/python_env.cpp:305:38 #1 0x559ede63acd4 in doris::BaseBackendService::get_python_envs(std::vector>&) /home/zcp/repo_center/doris_master/doris/be/src/service/backend_service.cpp:1316:47 #2 0x559edf99796a in doris::BackendServiceProcessor::process_get_python_envs(int, apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, void*) /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/BackendService.cpp:7789:13 #3 0x559edf94a69c in doris::BackendServiceProcessor::dispatchCall(apache::thrift::protocol::TProtocol*, apache::thrift::protocol::TProtocol*, std::__cxx11::basic_string, std::allocator> const&, int, void*) /home/zcp/repo_center/doris_master/doris/gensrc/build/gen_cpp/BackendService.cpp:6466:3 #4 0x559eac6bad06 in apache::thrift::TDispatchProcessor::process(std::shared_ptr, std::shared_ptr, void*) /home/zcp/repo_center/doris_master/doris/thirdparty/installed/include/thrift/TDispatchProcessor.h:121:12 #5 0x559ee47c071a in apache::thrift::server::TConnectedClient::run() (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1c871a) #6 0x559ee47c1a45 in apache::thrift::server::TThreadedServer::TConnectedClientRunner::run() (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1c9a45) #7 0x559ee47c557f in apache::thrift::concurrency::Thread::threadMain(std::shared_ptr) (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd57f) #8 0x559ee47c52a5 in void std::__invoke_impl), std::shared_ptr>(std::__invoke_other, void (*&&)(std::shared_ptr), std::shared_ptr&&) (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd2a5) #9 0x559ee47c521c in std::__invoke_result), std::shared_ptr>::type std::__invoke), std::shared_ptr>(void (*&&)(std::shared_ptr), std::shared_ptr&&) (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd21c) #10 0x559ee47c51f1 in void std::thread::_Invoker), std::shared_ptr>>::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd1f1) #11 0x559ee47c51b4 in std::thread::_Invoker), std::shared_ptr>>::operator()() (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1cd1b4) #12 0x559ee47c4fd8 in std::thread::_State_impl), std::shared_ptr>>>::_M_run() (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x8a1ccfd8) #13 0x559ef61b5f3f in execute_native_thread_routine archive64.c #14 0x559eac535d26 in asan_thread_start(void*) (/mnt/hdd01/ci/doris-deploy-master-local/be/lib/doris_be+0x51f3dd26) #15 0x7fcba62bfac2 in start_thread nptl/pthread_create.c:442:8 #16 0x7fcba635184f misc/../sysdeps/unix/sysv/linux/x86_64/clone3.S:81 SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /home/zcp/repo_center/doris_master/doris/be/src/udf/python/python_env.cpp:305:38 ``` now: ```text Doris> show python packages in '3.19.18'; ERROR 1105 (HY000): errCode = 2, detailMessage = Failed to get python packages from any backend: [E42] Set 'python_venv_interpreter_paths' in be.conf to enable PythonUDF feature Doris> show python versions; ERROR 1105 (HY000): errCode = 2, detailMessage = Failed to get python envs from any backend: [E-236] Set 'python_venv_interpreter_paths' in be.conf to enable PythonUDF feature ```
1 parent dc52c63 commit 2a680f5

4 files changed

Lines changed: 37 additions & 24 deletions

File tree

‎be/src/udf/python/python_env.cpp‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -267,8 +267,8 @@ std::string VenvEnvScanner::to_string() const {
267267
return ss.str();
268268
}
269269

270-
Status PythonVersionManager::init(PythonEnvType env_type, const fs::path& python_root_path,
271-
const std::string& python_venv_interpreter_paths) {
270+
Status PythonEnvScannerHolder::init(PythonEnvType env_type, const fs::path& python_root_path,
271+
const std::string& python_venv_interpreter_paths) {
272272
switch (env_type) {
273273
case PythonEnvType::CONDA: {
274274
if (!fs::exists(python_root_path) || !fs::is_directory(python_root_path)) {
@@ -302,10 +302,10 @@ Status PythonVersionManager::init(PythonEnvType env_type, const fs::path& python
302302

303303
std::vector<TPythonEnvInfo> PythonVersionManager::env_infos_to_thrift() const {
304304
std::vector<TPythonEnvInfo> infos;
305-
const auto& envs = _env_scanner->get_envs();
305+
const auto& envs = this->get_envs();
306306
infos.reserve(envs.size());
307307

308-
const auto env_type_str = _python_env_type_to_string(_env_scanner->env_type());
308+
const auto env_type_str = _python_env_type_to_string(this->env_type());
309309
for (const auto& env : envs) {
310310
TPythonEnvInfo info;
311311
info.__set_env_name(env.env_name);

‎be/src/udf/python/python_env.h‎

Lines changed: 30 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <filesystem>
2323
#include <utility>
2424

25+
#include "common/exception.h"
2526
#include "common/status.h"
2627

2728
namespace doris {
@@ -137,6 +138,27 @@ class VenvEnvScanner : public PythonEnvScanner {
137138
std::vector<std::string> _interpreter_paths;
138139
};
139140

141+
// Holds a PythonEnvScanner instance and centralizes the initialization check
142+
// to avoid accessing the scanner when the Python UDF feature is disabled.
143+
// This class is intended for internal use by PythonVersionManager only.
144+
class PythonEnvScannerHolder {
145+
public:
146+
Status init(PythonEnvType env_type, const fs::path& python_root_path,
147+
const std::string& python_venv_interpreter_paths);
148+
149+
const PythonEnvScanner& get() const {
150+
if (!_env_scanner) {
151+
throw Exception(ErrorCode::NOT_INITIALIZED,
152+
"Set 'enable_python_udf_support = true' in be.conf to enable PythonUDF "
153+
"feature");
154+
}
155+
return *_env_scanner;
156+
}
157+
158+
private:
159+
std::unique_ptr<PythonEnvScanner> _env_scanner;
160+
};
161+
140162
class PythonVersionManager {
141163
public:
142164
static PythonVersionManager& instance() {
@@ -145,25 +167,27 @@ class PythonVersionManager {
145167
}
146168

147169
Status init(PythonEnvType env_type, const fs::path& python_root_path,
148-
const std::string& python_venv_interpreter_paths);
170+
const std::string& python_venv_interpreter_paths) {
171+
return _holder.init(env_type, python_root_path, python_venv_interpreter_paths);
172+
}
149173

150174
Status get_version(const std::string& runtime_version, PythonVersion* version) const {
151-
return _env_scanner->get_version(runtime_version, version);
175+
return _holder.get().get_version(runtime_version, version);
152176
}
153177

154-
const std::vector<PythonEnvironment>& get_envs() const { return _env_scanner->get_envs(); }
178+
const std::vector<PythonEnvironment>& get_envs() const { return _holder.get().get_envs(); }
155179

156-
PythonEnvType env_type() const { return _env_scanner->env_type(); }
180+
PythonEnvType env_type() const { return _holder.get().env_type(); }
157181

158-
std::string to_string() const { return _env_scanner->to_string(); }
182+
std::string to_string() const { return _holder.get().to_string(); }
159183

160184
std::vector<TPythonEnvInfo> env_infos_to_thrift() const;
161185

162186
std::vector<TPythonPackageInfo> package_infos_to_thrift(
163187
const std::vector<std::pair<std::string, std::string>>& packages) const;
164188

165189
private:
166-
std::unique_ptr<PythonEnvScanner> _env_scanner;
190+
PythonEnvScannerHolder _holder;
167191
};
168192

169193
// List installed pip packages for a given Python version.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.catalog.Column;
2121
import org.apache.doris.catalog.Env;
2222
import org.apache.doris.catalog.ScalarType;
23+
import org.apache.doris.common.AnalysisException;
2324
import org.apache.doris.common.ClientPool;
2425
import org.apache.doris.common.ErrorCode;
2526
import org.apache.doris.common.ErrorReport;
@@ -37,8 +38,6 @@
3738

3839
import com.google.common.collect.ImmutableMap;
3940
import com.google.common.collect.Lists;
40-
import org.apache.logging.log4j.LogManager;
41-
import org.apache.logging.log4j.Logger;
4241

4342
import java.util.ArrayList;
4443
import java.util.HashMap;
@@ -50,8 +49,6 @@
5049
* Shows pip packages installed for the given Python version, collected from all alive BEs.
5150
*/
5251
public class ShowPythonPackagesCommand extends ShowCommand {
53-
private static final Logger LOG = LogManager.getLogger(ShowPythonPackagesCommand.class);
54-
5552
private static final String[] TITLE_NAMES = {"Package", "Version"};
5653
private static final String[] TITLE_NAMES_INCONSISTENT = {"Package", "Version", "Consistent", "Backends"};
5754

@@ -116,8 +113,6 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc
116113
}
117114
allBePackages.add(pkgMap);
118115
beIdentifiers.add(backend.getHost() + ":" + backend.getBePort());
119-
} catch (Exception e) {
120-
LOG.warn("Failed to get python packages from backend[{}]", backend.getId(), e);
121116
} finally {
122117
if (ok) {
123118
ClientPool.backendPool.returnObject(address, client);
@@ -128,7 +123,7 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc
128123
}
129124

130125
if (allBePackages.isEmpty()) {
131-
return new ShowResultSet(getMetaData(), Lists.newArrayList());
126+
throw new AnalysisException("No alive backends found to get python packages");
132127
}
133128

134129
// Check consistency across BEs

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,6 @@
3737

3838
import com.google.common.collect.ImmutableMap;
3939
import com.google.common.collect.Lists;
40-
import org.apache.logging.log4j.LogManager;
41-
import org.apache.logging.log4j.Logger;
4240

4341
import java.util.ArrayList;
4442
import java.util.HashSet;
@@ -50,8 +48,6 @@
5048
* Shows Python versions available on all alive backends (intersection).
5149
*/
5250
public class ShowPythonVersionsCommand extends ShowCommand {
53-
private static final Logger LOG = LogManager.getLogger(ShowPythonVersionsCommand.class);
54-
5551
private static final String[] TITLE_NAMES = {
5652
"Version", "EnvName", "EnvType", "BasePath", "ExecutablePath"
5753
};
@@ -108,8 +104,6 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc
108104
} else {
109105
commonVersions.retainAll(versions);
110106
}
111-
} catch (Exception e) {
112-
LOG.warn("Failed to get python envs from backend[{}]", backend.getId(), e);
113107
} finally {
114108
if (ok) {
115109
ClientPool.backendPool.returnObject(address, client);
@@ -120,7 +114,7 @@ public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exc
120114
}
121115

122116
List<List<String>> rows = Lists.newArrayList();
123-
if (commonVersions != null && !allBeEnvs.isEmpty()) {
117+
if (commonVersions != null) {
124118
// Use envs from the first BE as reference, filtered to common versions
125119
for (TPythonEnvInfo env : allBeEnvs.get(0)) {
126120
if (commonVersions.contains(env.getFullVersion())) {

0 commit comments

Comments
 (0)