Skip to content

Commit 0cb9274

Browse files
djouallahclaude
andcommitted
Doris probe: 4.1.4.1 as FE + BE containers, full BE crash log
4.1.4.x has no all-in-one image. apache/doris#68103 keeps OneLake on the Hadoop path, so the BE crash there is the blocker; print the whole be.out and any JVM crash report so it can be filed. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 9faaa40 commit 0cb9274

1 file changed

Lines changed: 133 additions & 14 deletions

File tree

‎.github/scripts/candidate_engine.py‎

Lines changed: 133 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -193,24 +193,40 @@ def _wait_for_backend(self) -> None:
193193

194194
# BE log files inside the container, printed when the BE dies.
195195
be_logs: tuple[str, ...] = ()
196+
be_log_lines = 60
196197

197-
def recover(self) -> None:
198-
try:
199-
if any("true" in map(str, r) for r in self.sql("SHOW BACKENDS")):
200-
return
201-
except Exception: # noqa: BLE001 - FE down too: restart below
202-
pass
203-
_say(" BE not alive: its log, then a container restart")
198+
def containers(self) -> tuple[str, ...]:
199+
"""Every container the engine runs in; the BE's is the last."""
200+
return (CONTAINER,)
201+
202+
def be_crash_log(self) -> None:
204203
for log in self.be_logs:
205204
tail = subprocess.run(
206-
["docker", "exec", CONTAINER, "tail", "-n", "60", log],
205+
[
206+
"docker",
207+
"exec",
208+
self.containers()[-1],
209+
"tail",
210+
"-n",
211+
str(self.be_log_lines),
212+
log,
213+
],
207214
capture_output=True,
208215
text=True,
209216
check=False,
210217
)
211218
_say(f" --- {log}")
212-
_say((tail.stdout + tail.stderr)[-6000:])
213-
subprocess.run(["docker", "restart", CONTAINER], check=False)
219+
_say((tail.stdout + tail.stderr)[-20000:])
220+
221+
def recover(self) -> None:
222+
try:
223+
if any("true" in map(str, r) for r in self.sql("SHOW BACKENDS")):
224+
return
225+
except Exception: # noqa: BLE001 - FE down too: restart below
226+
pass
227+
_say(" BE not alive: its log, then a container restart")
228+
self.be_crash_log()
229+
subprocess.run(["docker", "restart", *self.containers()], check=False)
214230
self._c = None
215231
self._wait_for_backend()
216232

@@ -640,17 +656,120 @@ class Doris(StarRocks):
640656
client-secret keys Doris insists on for `azure.auth_type=OAuth2` are then scoped to a host
641657
nothing reads -- on the FE they are written AFTER the raw keys, so on OneLake's host they
642658
would win there.
659+
660+
NATIVE AZURE WILL NOT COVER ONELAKE. apache/doris#68103 (native Azure credentials, vended SAS)
661+
keeps "genuine Fabric OneLake locations" on this Hadoop path, so the BE crash with workload
662+
identity / SAS on that path is the blocker, not a missing feature.
663+
664+
4.1.4.x ships only FE and BE component images (no all-in-one), so the default runs two
665+
containers on their own network. An `all-in-one-*` CANDIDATE_IMAGE runs the single container
666+
StarRocks does.
643667
"""
644668

645669
name = "doris"
646-
# 4.1.4 has component images but no all-in-one yet.
647-
default_image = "apache/doris:all-in-one-4.1.3"
670+
default_image = "apache/doris:fe-4.1.4.1"
648671
list_namespaces = "SHOW DATABASES FROM onelake"
649-
# The image is tuned for CI fixtures (BE mem_limit 40%); the bench runner is 16 GB.
650-
run_args = ("-e", "BE_CONFIG_EXTRA=mem_limit = 80%")
651672
be_logs = tuple(
652673
f"/opt/apache-doris/be/log/{name}" for name in ("be.out", "be.WARNING", "be.INFO")
653674
)
675+
# be.out holds the crash header and stack; 60 lines cut the stack off (run 36401357218).
676+
be_log_lines = 250
677+
NETWORK = "doris"
678+
SUBNET = "172.20.80"
679+
FE_SERVERS = f"fe1:{SUBNET}.2:9010"
680+
BE_CONTAINER = f"{CONTAINER}-be"
681+
682+
@property
683+
def all_in_one(self) -> bool:
684+
return ":all-in-one" in self.image
685+
686+
@property
687+
def run_args(self) -> tuple[str, ...]:
688+
# The all-in-one image is tuned for CI fixtures (BE mem_limit 40%); the runner is 16 GB.
689+
return ("-e", "BE_CONFIG_EXTRA=mem_limit = 80%") if self.all_in_one else ()
690+
691+
def containers(self) -> tuple[str, ...]:
692+
return (CONTAINER,) if self.all_in_one else (CONTAINER, self.BE_CONTAINER)
693+
694+
def start(self) -> None:
695+
if self.all_in_one:
696+
super().start()
697+
return
698+
_keep_assertion_fresh()
699+
# The BE refuses to start with a small max_map_count or with swap on.
700+
subprocess.run(["sudo", "sysctl", "-w", "vm.max_map_count=2000000"], check=True)
701+
subprocess.run(["sudo", "swapoff", "-a"], check=True)
702+
subprocess.run(
703+
["docker", "network", "create", "--subnet", f"{self.SUBNET}.0/24", self.NETWORK],
704+
check=True,
705+
)
706+
mount = f"{ASSERTION_DIR}:{Path(ASSERTION_IN_CONTAINER).parent}:ro"
707+
be_image = self.image.replace(":fe-", ":be-")
708+
for name, ip, image, env, ports in (
709+
(
710+
CONTAINER,
711+
f"{self.SUBNET}.2",
712+
self.image,
713+
("FE_ID=1",),
714+
("9030:9030", "8030:8030"),
715+
),
716+
(
717+
self.BE_CONTAINER,
718+
f"{self.SUBNET}.3",
719+
be_image,
720+
(f"BE_ADDR={self.SUBNET}.3:9050",),
721+
("8040:8040",),
722+
),
723+
):
724+
subprocess.run(
725+
[
726+
"docker",
727+
"run",
728+
"-d",
729+
"--name",
730+
name,
731+
"--network",
732+
self.NETWORK,
733+
"--ip",
734+
ip,
735+
"-v",
736+
mount,
737+
"-e",
738+
f"FE_SERVERS={self.FE_SERVERS}",
739+
*(a for e in env for a in ("-e", e)),
740+
*(a for p in ports for a in ("-p", f"127.0.0.1:{p}")),
741+
image,
742+
],
743+
check=True,
744+
)
745+
digest = subprocess.run(
746+
["docker", "image", "inspect", "--format", "{{index .RepoDigests 0}}", image],
747+
capture_output=True,
748+
text=True,
749+
check=False,
750+
).stdout.strip()
751+
_say(f"image {image} {digest}")
752+
self._wait_for_backend()
753+
754+
def be_crash_log(self) -> None:
755+
super().be_crash_log()
756+
# A SIGSEGV inside the JNI hadoop-azure call may leave a JVM crash report instead.
757+
report = subprocess.run(
758+
[
759+
"docker",
760+
"exec",
761+
self.containers()[-1],
762+
"sh",
763+
"-c",
764+
"for f in $(find /opt/apache-doris -name 'hs_err_pid*.log' 2>/dev/null); "
765+
'do echo "--- $f"; head -n 120 "$f"; done',
766+
],
767+
capture_output=True,
768+
text=True,
769+
check=False,
770+
)
771+
if report.stdout.strip():
772+
_say(report.stdout[-8000:])
654773

655774
HOST = ONELAKE_DFS
656775
UNUSED_HOST = "unused.dfs.core.windows.net"

0 commit comments

Comments
 (0)