Skip to content

Commit 9140a99

Browse files
committed
fix(perf): bound harness child processes and skip RSS on unsupported platforms
Add a cancellable timeout to run-process and thread --package-runner-timeout-ms through the package URL probe, install, pnpm pack, and git rev-parse flows so a stalled child cannot outlive the deadline; give the curl probe and download explicit connect and read limits. measure-memory now warns once and skips gracefully when /usr/bin/time is unavailable (unsupported platforms) instead of throwing and aborting the entire benchmark run. Co-authored-by: Codesmith <[email protected]>
1 parent bf40013 commit 9140a99

4 files changed

Lines changed: 44 additions & 21 deletions

File tree

‎apps/ccusage/scripts/compare-pr-performance_test.clj‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,18 @@
123123
(is (.contains markdown
124124
"| Command | Input | Base median | PR median | PR vs base | Base peak RSS | PR peak RSS | PR&#x2f;base RSS | Base throughput | PR throughput |"))))
125125

126+
(deftest process-timeout
127+
(testing "a stalled child is terminated and reported as timed out"
128+
(let [result (run-process ["sleep" "10"] {:timeout-ms 200})]
129+
(is (:timed-out? result))
130+
(is (not (zero? (:exit result))))
131+
(is (.contains (:err result) "timed out")))))
132+
133+
(deftest memory-skips-on-unsupported-platform
134+
(testing "unsupported platforms warn and skip instead of aborting the run"
135+
(with-redefs [platform-name (constantly "win32")]
136+
(is (nil? (measure-memory {:argv ["true"] :env {}} {:runs 1 :label "test"}))))))
137+
126138
(let [{:keys [fail error]} (apply run-tests ['compare-pr-performance-test])]
127139
(when (pos? (+ fail error))
128140
(System/exit 1)))

‎apps/ccusage/scripts/compare_pr_performance/benchmark.clj‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,17 @@
2525
nil))
2626
(defn measure-memory [command {:keys [runs label]}]
2727
(when (pos? runs)
28-
(let [samples (keep (fn [_]
29-
(if-let [argv (timed-command command)]
28+
(if-let [argv (timed-command command)]
29+
(let [samples (keep (fn [_]
3030
(let [result (run-process argv {:env (:env command)})]
3131
(if (zero? (:exit result))
3232
(try (parse-peak-rss (:err result))
3333
(catch Exception e (write-progress (str label " peak RSS skipped: " (ex-message e))) nil))
34-
(do (write-progress (format "%s peak RSS skipped after exit %d: %s" label (:exit result) (:err result))) nil)))
35-
(fail! (str "Peak RSS measurement is not supported on " (platform-name)) {:kind :unsupported-platform})))
36-
(range runs))]
37-
(when (seq samples)
38-
{:peak-rss-bytes (nth (vec (sort samples)) (quot (count samples) 2)) :samples (count samples)}))))
34+
(do (write-progress (format "%s peak RSS skipped after exit %d: %s" label (:exit result) (:err result))) nil))))
35+
(range runs))]
36+
(when (seq samples)
37+
{:peak-rss-bytes (nth (vec (sort samples)) (quot (count samples) 2)) :samples (count samples)}))
38+
(do (write-progress (str label " peak RSS skipped: not supported on " (platform-name))) nil))))
3939

4040
(defn read-hyperfine-results [export-path expected]
4141
(let [data (try (json/parse-string (slurp export-path) true)

‎apps/ccusage/scripts/compare_pr_performance/main.clj‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
{:skip-reason (str "Base package URL was not ready before " (format-duration (:package-runner-timeout-ms options))
1212
". Fixture performance comparison requires a base package when --base-dir is not provided.")
1313
:base-package-url (:base-package-url options) :base-sha (:base-sha options)
14-
:head-runtime (parse-head-runtime (:head-runtime options)) :head-sha (git-sha head-dir)}
14+
:head-runtime (parse-head-runtime (:head-runtime options)) :head-sha (git-sha head-dir (:package-runner-timeout-ms options))}
1515
(let [base-native (when base-install (installed-native-package-bin-entry (path install-root "base-package")))
1616
head-native (when head-install (installed-native-package-bin-entry (path install-root "head-package")))
1717
runtime (parse-head-runtime (:head-runtime options))]
@@ -26,7 +26,8 @@
2626
:base-runtime-description (when base-install "Base runs the published `ccusage` package from `pkg.pr.new`, installed before measurement")
2727
:head-runtime-description (cond (and (= runtime "rust") head-native) "PR runs the published native `ccusage` binary from `pkg.pr.new`, installed before measurement"
2828
(and (= runtime "package") head-install) "PR runs the published `ccusage` package from `pkg.pr.new`, installed before measurement")
29-
:base-sha (or (:base-sha options) (when base-dir (git-sha base-dir))) :head-sha (git-sha head-dir)})))))
29+
:base-sha (or (:base-sha options) (when base-dir (git-sha base-dir (:package-runner-timeout-ms options))))
30+
:head-sha (git-sha head-dir (:package-runner-timeout-ms options))})))))
3031

3132
(defn run-benchmark-suites [context]
3233
(let [common (select-keys context [:base-bin-entry :base-runtime-description :base-sha :head-bin-entry :head-dir
@@ -53,12 +54,12 @@
5354
{:base-native-package-binary (optional-file-size (:base-native-bin-entry context))
5455
:base-package (if (= :remote (base-package-size-source context))
5556
(remote-tarball-size (:base-package-url context))
56-
(packed-tarball-size (:base-dir context)))
57+
(packed-tarball-size (:base-dir context) (:package-runner-timeout-ms context)))
5758
:base-rust-binary (when (:base-dir context) (optional-file-size (rust-binary-entry (:base-dir context))))
5859
:head-native-package-binary (optional-file-size (:head-native-bin-entry context))
5960
:head-package (if (= :remote (head-package-size-source context))
6061
(remote-tarball-size (:head-package-url context))
61-
(packed-tarball-size (:head-dir context)))
62+
(packed-tarball-size (:head-dir context) (:package-runner-timeout-ms context)))
6263
:head-rust-binary (optional-file-size (rust-binary-entry (:head-dir context)))})
6364

6465
(defn -main [& argv]

‎apps/ccusage/scripts/compare_pr_performance/system.clj‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,19 @@
11
(defn run-process
22
([argv] (run-process argv {}))
3-
([argv {:keys [dir env inherit?] :or {env {}}}]
3+
([argv {:keys [dir env inherit? timeout-ms] :or {env {}}}]
44
(let [opts (cond-> {:env (merge (into {} (System/getenv)) env)}
55
dir (assoc :dir (str dir))
66
inherit? (assoc :out :inherit :err :inherit)
77
(not inherit?) (assoc :out :string :err :string))
8-
result @(process/process argv opts)]
9-
{:exit (:exit result) :out (or (:out result) "") :err (or (:err result) "")})))
8+
proc (process/process argv opts)
9+
result (if (and timeout-ms (pos? timeout-ms))
10+
(let [outcome (deref proc timeout-ms ::timeout)]
11+
(if (= outcome ::timeout)
12+
(do (process/destroy-tree proc) (deref proc) {:exit 124 :timed-out? true})
13+
outcome))
14+
@proc)]
15+
(cond-> {:exit (:exit result) :out (or (:out result) "") :err (or (:err result) "")}
16+
(:timed-out? result) (assoc :timed-out? true :err (str "timed out after " timeout-ms "ms"))))))
1017

1118
(defn run-checked [argv options label]
1219
(let [result (run-process argv options)]
@@ -62,10 +69,12 @@
6269
bin))))
6370

6471
(defn write-progress [message] (binding [*out* *err*] (println (str "[ccusage-perf] " message))))
65-
(defn git-sha [directory]
66-
(str/trim (:out (run-checked ["git" "rev-parse" "HEAD"] {:dir directory} "git rev-parse HEAD"))))
72+
(def curl-timeout-flags ["--connect-timeout" "10" "--max-time" "60"])
73+
(defn git-sha [directory timeout-ms]
74+
(str/trim (:out (run-checked ["git" "rev-parse" "HEAD"] {:dir directory :timeout-ms timeout-ms} "git rev-parse HEAD"))))
6775
(defn package-url-ready? [url]
68-
(zero? (:exit (run-process ["curl" "--fail" "--head" "--location" "--silent" "--output" "/dev/null" url]))))
76+
(zero? (:exit (run-process (into ["curl" "--fail" "--head" "--location" "--silent"]
77+
(concat curl-timeout-flags ["--output" "/dev/null" url]))))))
6978
(defn wait-for-package-url [url timeout-ms]
7079
(let [deadline (+ (System/currentTimeMillis) timeout-ms)]
7180
(loop []
@@ -83,7 +92,7 @@
8392
(json/generate-string {:private true :dependencies {:ccusage url}} {:pretty true}))
8493
(write-progress (str label " package install started: " url))
8594
(let [started (System/currentTimeMillis)
86-
result (run-process ["bun" "install" "--no-progress"] {:dir install-dir})
95+
result (run-process ["bun" "install" "--no-progress"] {:dir install-dir :timeout-ms timeout-ms})
8796
elapsed (- (System/currentTimeMillis) started)]
8897
(when-not (zero? (:exit result))
8998
(fail! (str label " package install failed: "
@@ -98,13 +107,13 @@
98107
(defn packed-tarball-path [destination filename]
99108
(if (fs/absolute? filename) filename (path destination filename)))
100109

101-
(defn packed-tarball-size [repo-dir]
110+
(defn packed-tarball-size [repo-dir timeout-ms]
102111
(let [package-dir (path repo-dir "apps" "ccusage")
103112
package-json-path (path package-dir "package.json")
104113
original (slurp package-json-path)
105114
destination (temp-dir "ccusage-pack.")]
106115
(try
107-
(let [result (run-process ["pnpm" "pack" "--json" "--pack-destination" (str destination)] {:dir package-dir})]
116+
(let [result (run-process ["pnpm" "pack" "--json" "--pack-destination" (str destination)] {:dir package-dir :timeout-ms timeout-ms})]
108117
(when-not (zero? (:exit result))
109118
(fail! (str "pnpm pack failed: " (str/trim (or (not-empty (:err result)) (:out result))))
110119
{:kind :package-metadata-failed :exit (:exit result)}))
@@ -120,7 +129,8 @@
120129
(defn remote-tarball-size [url]
121130
(let [output (fs/create-temp-file {:prefix "ccusage-package."})]
122131
(try
123-
(run-checked ["curl" "--fail" "--location" "--silent" "--output" (str output) url]
132+
(run-checked (into ["curl" "--fail" "--location" "--silent"]
133+
(concat curl-timeout-flags ["--output" (str output) url]))
124134
{} (str "Failed to fetch " url))
125135
(optional-file-size output)
126136
(finally (fs/delete-if-exists output)))))

0 commit comments

Comments
 (0)