|
| 1 | +# Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +# or more contributor license agreements. See the NOTICE file |
| 3 | +# distributed with this work for additional information |
| 4 | +# regarding copyright ownership. The ASF licenses this file |
| 5 | +# to you under the Apache License, Version 2.0 (the |
| 6 | +# "License"); you may not use this file except in compliance |
| 7 | +# with the License. You may obtain a copy of the License at |
| 8 | +# |
| 9 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +# |
| 11 | +# Unless required by applicable law or agreed to in writing, |
| 12 | +# software distributed under the License is distributed on an |
| 13 | +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +# KIND, either express or implied. See the License for the |
| 15 | +# specific language governing permissions and limitations |
| 16 | +# under the License. |
| 17 | + |
| 18 | +"""Summarize which writer ran the Iceberg writes of an Iceberg Spark test run. |
| 19 | +
|
| 20 | +The Iceberg Spark test jobs set COMET_ICEBERG_WRITE_REPORT_DIR, so Comet's |
| 21 | +IcebergWriteReportListener appends one JSON line per Iceberg write to a file |
| 22 | +in that directory. This prints how many writes ran on Comet's native writer, |
| 23 | +how many Comet's split operator left on Iceberg's JVM writer and why, and how |
| 24 | +many Spark planned without Comet's split operator: |
| 25 | +
|
| 26 | + python3 dev/ci/summarize-iceberg-writes.py --title "iceberg-spark shard 1" DIR... |
| 27 | +
|
| 28 | +Files under a directory named like a shard artifact (...-shard-N-attempt-M) |
| 29 | +are counted only for the latest attempt of each shard, so a rerun of failed |
| 30 | +jobs does not count a shard twice. The latest attempt is the newest such |
| 31 | +directory, whether or not it holds any report files, so a rerun that recorded |
| 32 | +no writes is reported as missing rather than replaced by an earlier attempt. |
| 33 | +The summary is also appended to $GITHUB_STEP_SUMMARY when that is set. It |
| 34 | +never fails the job. |
| 35 | +""" |
| 36 | + |
| 37 | +import argparse |
| 38 | +from collections import Counter |
| 39 | +import json |
| 40 | +import os |
| 41 | +from pathlib import Path |
| 42 | +import re |
| 43 | + |
| 44 | + |
| 45 | +SHARD_ATTEMPT = re.compile(r"-shard-(\d+)-attempt-(\d+)$") |
| 46 | +WRITERS = [ |
| 47 | + ("native", "Comet native writer"), |
| 48 | + ("jvm", "Iceberg JVM writer under Comet's split operator"), |
| 49 | + ("spark", "Spark V2 write, not planned by Comet's split operator"), |
| 50 | +] |
| 51 | +TOP_REASONS = 20 |
| 52 | + |
| 53 | + |
| 54 | +def shard_attempt(path): |
| 55 | + """The (shard, attempt) of the shard artifact directory holding path, or None.""" |
| 56 | + for part in path.parts: |
| 57 | + match = SHARD_ATTEMPT.search(part) |
| 58 | + if match: |
| 59 | + return int(match.group(1)), int(match.group(2)) |
| 60 | + return None |
| 61 | + |
| 62 | + |
| 63 | +def report_files(roots): |
| 64 | + """Every report file under roots, keeping only the latest attempt of each shard. |
| 65 | +
|
| 66 | + Returns the files and the (shard, attempt) pairs whose latest attempt holds no report file. |
| 67 | + The latest attempt comes from the artifact directories rather than the report files, because |
| 68 | + an attempt that recorded no writes still uploads its shard inventory and test reports. |
| 69 | + """ |
| 70 | + latest = {} |
| 71 | + files = [] |
| 72 | + for root in map(Path, roots): |
| 73 | + for path in [root, *sorted(root.rglob("*"))]: |
| 74 | + key = shard_attempt(path) |
| 75 | + if key: |
| 76 | + latest[key[0]] = max(latest.get(key[0], 0), key[1]) |
| 77 | + if path.suffix == ".jsonl" and path.is_file(): |
| 78 | + files.append((path, key)) |
| 79 | + kept = [(path, key) for path, key in files if key is None or latest[key[0]] == key[1]] |
| 80 | + reported = {key for _, key in kept} |
| 81 | + missing = sorted(key for key in latest.items() if key not in reported) |
| 82 | + return [path for path, _ in kept], missing |
| 83 | + |
| 84 | + |
| 85 | +def load(roots): |
| 86 | + files, missing = report_files(roots) |
| 87 | + writes = [] |
| 88 | + for path in files: |
| 89 | + for line in path.read_text(encoding="utf-8").splitlines(): |
| 90 | + if line.strip(): |
| 91 | + writes.append(json.loads(line)) |
| 92 | + return writes, missing |
| 93 | + |
| 94 | + |
| 95 | +def cell(text): |
| 96 | + return " ".join(text.split()).replace("|", "\\|") |
| 97 | + |
| 98 | + |
| 99 | +def summarize(title, writes, missing=()): |
| 100 | + lines = [f"### Iceberg writes: {title}", ""] |
| 101 | + for shard, attempt in missing: |
| 102 | + lines += [ |
| 103 | + f"Shard {shard} recorded no Iceberg writes in its latest attempt ({attempt}), " |
| 104 | + "so none of its writes are counted below.", |
| 105 | + "", |
| 106 | + ] |
| 107 | + if not writes: |
| 108 | + lines.append( |
| 109 | + "No Iceberg writes were recorded. Either the target ran none or " |
| 110 | + "COMET_ICEBERG_WRITE_REPORT_DIR did not reach the test JVMs." |
| 111 | + ) |
| 112 | + return "\n".join(lines) + "\n" |
| 113 | + |
| 114 | + total = len(writes) |
| 115 | + by_writer = Counter(w["writer"] for w in writes) |
| 116 | + lines += ["| Writer | Writes | Share |", "| --- | ---: | ---: |"] |
| 117 | + for key, label in WRITERS: |
| 118 | + count = by_writer.get(key, 0) |
| 119 | + lines.append(f"| {label} | {count} | {100.0 * count / total:.1f}% |") |
| 120 | + lines += [f"| Total | {total} | |", ""] |
| 121 | + failed = sum(1 for w in writes if w.get("failed")) |
| 122 | + if failed: |
| 123 | + lines += [f"{failed} of the {total} writes ran in queries that failed.", ""] |
| 124 | + |
| 125 | + reasons = Counter() |
| 126 | + for w in writes: |
| 127 | + if w["writer"] == "jvm": |
| 128 | + for reason in w.get("reasons") or ["(no reason recorded)"]: |
| 129 | + reasons[reason] += 1 |
| 130 | + if reasons: |
| 131 | + lines += [ |
| 132 | + "#### Why the split operator kept the JVM writer", |
| 133 | + "", |
| 134 | + "A write can have several reasons, so the counts can add up to more than the " |
| 135 | + "JVM writes.", |
| 136 | + "", |
| 137 | + "| Writes | Reason |", |
| 138 | + "| ---: | --- |", |
| 139 | + ] |
| 140 | + for reason, count in reasons.most_common(TOP_REASONS): |
| 141 | + lines.append(f"| {count} | {cell(reason)} |") |
| 142 | + if len(reasons) > TOP_REASONS: |
| 143 | + lines.append(f"| | and {len(reasons) - TOP_REASONS} more reasons |") |
| 144 | + lines.append("") |
| 145 | + |
| 146 | + operators = Counter(w["node"] for w in writes if w["writer"] == "spark") |
| 147 | + if operators: |
| 148 | + lines += ["#### Spark V2 writes by operator", "", "| Writes | Operator |", "| ---: | --- |"] |
| 149 | + for node, count in operators.most_common(): |
| 150 | + lines.append(f"| {count} | {cell(node)} |") |
| 151 | + lines.append("") |
| 152 | + |
| 153 | + return "\n".join(lines) + "\n" |
| 154 | + |
| 155 | + |
| 156 | +def main(): |
| 157 | + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) |
| 158 | + parser.add_argument("--title", required=True, help="heading for the summary") |
| 159 | + parser.add_argument("roots", nargs="+", help="directories holding the report files") |
| 160 | + args = parser.parse_args() |
| 161 | + |
| 162 | + summary = summarize(args.title, *load(args.roots)) |
| 163 | + print(summary) |
| 164 | + step_summary = os.environ.get("GITHUB_STEP_SUMMARY") |
| 165 | + if step_summary: |
| 166 | + with open(step_summary, "a", encoding="utf-8") as out: |
| 167 | + out.write(summary + "\n") |
| 168 | + |
| 169 | + |
| 170 | +if __name__ == "__main__": |
| 171 | + main() |
0 commit comments