Skip to content

Commit c7a3306

Browse files
committed
Merge remote-tracking branch 'upstream/master' into HEAD
2 parents 54ad06e + af9631d commit c7a3306

File tree

90 files changed

+3225
-722
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

90 files changed

+3225
-722
lines changed

ci/jobs/scripts/workflow_hooks/pr_description.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ def check_labels(category, info):
172172

173173
if info.pr_number:
174174
changed_files_str = Shell.get_output(
175-
"git diff --name-only $(git merge-base HEAD origin/master) --cached"
175+
"git diff --name-only $(git merge-base HEAD origin/master) --cached", strict=True
176176
)
177177
if "contrib/" in changed_files_str:
178178
pr_labels_to_add.append(Labels.SUBMODULE_CHANGED)

ci/praktika/_environment.py

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -199,11 +199,15 @@ def from_env(cls) -> "_Environment":
199199
)
200200

201201
def get_s3_prefix(self, latest=False):
202-
return self.get_s3_prefix_static(self.PR_NUMBER, self.SHA, latest)
202+
return self.get_s3_prefix_static(self.PR_NUMBER, self.BRANCH, self.SHA, latest)
203203

204204
@classmethod
205-
def get_s3_prefix_static(cls, pr_number, sha, latest=False):
206-
prefix = f"{pr_number}"
205+
def get_s3_prefix_static(cls, pr_number, branch, sha, latest=False):
206+
assert pr_number or branch
207+
if pr_number:
208+
prefix = f"PRs/{pr_number}"
209+
else:
210+
prefix = f"REFs/{branch}"
207211
assert sha or latest
208212
assert pr_number >= 0
209213
if latest:

ci/praktika/cache.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ def __init__(self):
3939
self.success = {} # type Dict[str, Any]
4040

4141
@classmethod
42-
def push_success_record(cls, job_name, job_digest, sha):
42+
def push_success_record(cls, job_name, job_digest, sha, if_not_exist):
4343
type_ = Cache.CacheRecord.Type.SUCCESS
4444
record = Cache.CacheRecord(
4545
type=type_,
@@ -50,10 +50,12 @@ def push_success_record(cls, job_name, job_digest, sha):
5050
assert (
5151
Settings.CACHE_S3_PATH
5252
), f"Setting CACHE_S3_PATH must be defined with enabled CI Cache"
53-
record_path = f"{Settings.CACHE_S3_PATH}/v{Settings.CACHE_VERSION}/{Utils.normalize_string(job_name)}/{job_digest}"
53+
record_path = f"{Settings.CACHE_S3_PATH}/v{Settings.CACHE_VERSION}/{Utils.normalize_string(job_name)}/{job_digest}/{type_}"
5454
record_file = Path(Settings.TEMP_DIR) / type_
5555
record.dump(record_file)
56-
S3.copy_file_to_s3(s3_path=record_path, local_path=record_file)
56+
S3.put(
57+
s3_path=record_path, local_path=record_file, if_none_matched=if_not_exist
58+
)
5759
record_file.unlink()
5860

5961
def fetch_success(self, job_name, job_digest):

ci/praktika/hook_cache.py

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,14 @@ def fetch_record(job_name, job_digest, cache_):
7575
if result: # If a record was found, add it to the fetched list
7676
fetched_records.append(result)
7777

78+
env = _Environment.get()
7879
# Step 2: Apply the fetched records sequentially
7980
for job_name, record in fetched_records:
8081
assert Utils.normalize_string(job_name) not in workflow_config.cache_success
82+
if workflow.is_event_push() and record.branch != env.BRANCH:
83+
# TODO: make this behaviour configurable?
84+
print(f"NOTE: Result for [{job_name}] cached from branch [{record.branch}] - skip for workflow with event=PUSH")
85+
continue
8186
workflow_config.cache_success.append(job_name)
8287
workflow_config.cache_success_base64.append(Utils.to_base64(job_name))
8388
workflow_config.cache_jobs[job_name] = record
@@ -107,7 +112,7 @@ def fetch_record(job_name, job_digest, cache_):
107112
)
108113

109114
print(f"Write config to GH's job output")
110-
with open(_Environment.get().JOB_OUTPUT_STREAM, "a", encoding="utf8") as f:
115+
with open(env.JOB_OUTPUT_STREAM, "a", encoding="utf8") as f:
111116
print(
112117
f"DATA={workflow_config.to_json()}",
113118
file=f,
@@ -136,7 +141,9 @@ def pre_run(cls, _workflow, _job, _required_artifacts=None):
136141
record = runtime_config.cache_artifacts[artifact.name]
137142
print(f"Reuse artifact [{artifact.name}] from [{record}]")
138143
path_prefixes.append(
139-
env.get_s3_prefix_static(record.pr_number, record.sha)
144+
env.get_s3_prefix_static(
145+
record.pr_number, record.branch, record.sha
146+
)
140147
)
141148
else:
142149
path_prefixes.append(env.get_s3_prefix())
@@ -154,4 +161,10 @@ def post_run(cls, workflow, job):
154161
# cache is enabled, and it's a job that supposed to be cached (has defined digest config)
155162
workflow_runtime = RunConfig.from_fs(workflow.name)
156163
job_digest = workflow_runtime.digest_jobs[job.name]
157-
Cache.push_success_record(job.name, job_digest, workflow_runtime.sha)
164+
# if_not_exist=workflow.is_event_pull_request() - to not overwrite record from "push" workflow, as it can reuse only from push, "pull_request" - from both
165+
Cache.push_success_record(
166+
job.name,
167+
job_digest,
168+
workflow_runtime.sha,
169+
if_not_exist=workflow.is_event_pull_request(),
170+
)

ci/praktika/hook_html.py

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,21 @@ def dump(cls, commits):
7171
with open(cls.file_name(), "w", encoding="utf8") as f:
7272
json.dump(commits_, f)
7373

74+
@classmethod
75+
def get_s3_path(cls):
76+
env = _Environment.get()
77+
if env.PR_NUMBER:
78+
s3suffix = f"PRs/{env.PR_NUMBER}"
79+
else:
80+
assert env.BRANCH
81+
s3suffix = f"REFs/{env.BRANCH}"
82+
return f"{Settings.HTML_S3_PATH}/{s3suffix}"
83+
7484
@classmethod
7585
def pull_from_s3(cls):
7686
local_path = Path(cls.file_name())
7787
file_name = local_path.name
78-
env = _Environment.get()
79-
s3_path = f"{Settings.HTML_S3_PATH}/{env.PR_NUMBER}/{file_name}"
88+
s3_path = f"{cls.get_s3_path()}/{file_name}"
8089
if not S3.copy_file_from_s3(s3_path=s3_path, local_path=local_path):
8190
print(f"WARNING: failed to cp file [{s3_path}] from s3")
8291
return []
@@ -88,8 +97,7 @@ def push_to_s3(cls, commits):
8897
cls.dump(commits)
8998
local_path = Path(cls.file_name())
9099
file_name = local_path.name
91-
env = _Environment.get()
92-
s3_path = f"{Settings.HTML_S3_PATH}/{env.PR_NUMBER}/{file_name}"
100+
s3_path = f"{cls.get_s3_path()}/{file_name}"
93101
if not S3.copy_file_to_s3(s3_path=s3_path, local_path=local_path, text=True):
94102
print(f"WARNING: failed to cp file [{local_path}] to s3")
95103

ci/praktika/info.py

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -81,22 +81,29 @@ def is_local_run(self):
8181

8282
def get_report_url(self, latest=False):
8383
sha = self.env.SHA
84-
if self.env.PR_NUMBER and latest:
84+
if latest:
8585
sha = "latest"
86-
return self.get_specific_report_url(pr_number=self.env.PR_NUMBER, sha=sha)
86+
return self.get_specific_report_url(
87+
pr_number=self.env.PR_NUMBER, branch=self.env.BRANCH, sha=sha
88+
)
8789

8890
def dump(self):
8991
self.env.dump()
9092

91-
def get_specific_report_url(self, pr_number, sha, job_name=""):
93+
def get_specific_report_url(self, pr_number, branch, sha, job_name=""):
9294
from praktika.settings import Settings
9395

96+
if pr_number:
97+
ref_param = f"PR={pr_number}"
98+
else:
99+
assert branch
100+
ref_param = f"REF={branch}"
94101
path = Settings.HTML_S3_PATH
95102
for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items():
96103
if bucket in path:
97104
path = path.replace(bucket, endpoint)
98105
break
99-
res = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?PR={pr_number}&sha={sha}&name_0={urllib.parse.quote(self.env.WORKFLOW_NAME, safe='')}"
106+
res = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?{ref_param}&sha={sha}&name_0={urllib.parse.quote(self.env.WORKFLOW_NAME, safe='')}"
100107
if job_name:
101108
res += f"&name_1={urllib.parse.quote(job_name, safe='')}"
102109
return res

ci/praktika/json.html

Lines changed: 28 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -534,54 +534,6 @@
534534
statusContainer.appendChild(keyValuePair);
535535
}
536536

537-
function navigatePath(jsonObj, nameArray) {
538-
let baseParams = new URLSearchParams(window.location.search);
539-
let keysToDelete = [];
540-
baseParams.forEach((value, key) => {
541-
if (key.startsWith('name_')) {
542-
keysToDelete.push(key); // Collect the keys to delete
543-
}
544-
});
545-
keysToDelete.forEach((key) => baseParams.delete(key));
546-
let pathNames = [];
547-
let pathLinks = [];
548-
let currentObj = jsonObj;
549-
550-
// Add the first entry (workflow level)
551-
baseParams.set(`name_0`, nameArray[0]);
552-
pathNames.push(nameArray[0]);
553-
pathLinks.push(`<span class="separator">/</span><a href="${window.location.pathname}?${baseParams.toString()}">${nameArray[0]}</a>`);
554-
// Add the second entry (job level)
555-
if (nameArray.length > 1) {
556-
baseParams.set(`name_1`, nameArray[1]);
557-
pathNames.push(nameArray[1]);
558-
pathLinks.push(`<span class="separator">/</span><a href="${window.location.pathname}?${baseParams.toString()}">${nameArray[1]}</a>`);
559-
}
560-
// Iterate through the nameArray starting at index 0
561-
for (const [index, name] of nameArray.entries()) {
562-
if (index === 0 || index === 1) continue;
563-
if (currentObj && Array.isArray(currentObj.results)) {
564-
const nextResult = currentObj.results.find(result => result.name === name);
565-
if (nextResult) {
566-
baseParams.set(`name_${index}`, nextResult.name);
567-
pathNames.push(nextResult.name); // Correctly push nextResult name, not currentObj.name
568-
pathLinks.push(`<span class="separator">/</span><a href="${window.location.pathname}?${baseParams.toString()}">${nextResult.name}</a>`);
569-
currentObj = nextResult; // Move to the next object in the hierarchy
570-
} else {
571-
console.error(`Name "${name}" not found in results array.`);
572-
return null; // Name not found in results array
573-
}
574-
} else {
575-
console.error(`Current object is not structured as expected.`);
576-
return null; // Current object is not structured as expected
577-
}
578-
}
579-
const footerLeft = document.querySelector('#footer .left');
580-
footerLeft.innerHTML = pathLinks.join('');
581-
582-
return currentObj;
583-
}
584-
585537
// Define the fixed columns globally, so both functions can use it
586538
const columns = ['name', 'status', 'start_time', 'duration', 'info'];
587539

@@ -696,7 +648,7 @@
696648
populateTableRows(tbody, results, columns, nest_level);
697649
}
698650

699-
async function fetchJson(PR, sha, name) {
651+
async function fetchJson(suffix, sha, name) {
700652
const infoElement = document.getElementById('info-container');
701653
const task = name
702654
.toLowerCase()
@@ -706,7 +658,7 @@
706658

707659
// Construct the URL dynamically based on PR, sha, and task
708660
const baseUrl = window.location.origin + window.location.pathname.replace('/json.html', '');
709-
const path = `${baseUrl}/${encodeURIComponent(PR)}/${encodeURIComponent(sha)}/result_${task}.json`;
661+
const path = `${baseUrl}/${suffix}/${encodeURIComponent(sha)}/result_${task}.json`;
710662

711663
try {
712664
const response = await fetch(path, { cache: "no-cache" });
@@ -716,8 +668,7 @@
716668
}
717669

718670
//const lastModifiedTime = response.headers.get('Last-Modified');
719-
const jsonData = await response.json();
720-
return jsonData;
671+
return await response.json();
721672
} catch (error) {
722673
console.error('Error loading Results:', error);
723674
if (infoElement) {
@@ -741,9 +692,9 @@
741692
else if (link.includes('/actions/runs/')) runtimeCache.runLink = link;
742693
}
743694

744-
async function fetch0Json(PR, sha, name) {
695+
async function fetch0Json(suffix, sha, name) {
745696
try {
746-
const resultData = await fetchJson(PR, sha, name);
697+
const resultData = await fetchJson(suffix, sha, name);
747698
runtimeCache.json0 = resultData;
748699
if (Array.isArray(resultData.links)) {
749700
resultData.links.forEach(storeLink);
@@ -753,10 +704,9 @@
753704
}
754705
}
755706

756-
async function fetch1Json(PR, sha, name) {
707+
async function fetch1Json(suffix, sha, name) {
757708
try {
758-
const resultData = await fetchJson(PR, sha, name);
759-
runtimeCache.json1 = resultData;
709+
runtimeCache.json1 = await fetchJson(suffix, sha, name);
760710
} catch (error) {
761711
console.error('Error fetching JSON1:', error);
762712
}
@@ -859,17 +809,25 @@
859809
footerLeft.innerHTML = pathLinks.join('');
860810
}
861811

862-
async function renderResults(PR, sha, nameParams) {
812+
async function renderResults(PR, REF, sha, nameParams) {
863813
const baseUrl = window.location.origin + window.location.pathname.replace('/json.html', '');
864-
if (PR>0){
865-
await loadCommitsArray(`${baseUrl}/${encodeURIComponent(PR)}/commits.json`)
814+
let suffix = '';
815+
if (PR) {
816+
suffix = `PRs/${encodeURIComponent(PR)}`;
817+
} else if (REF) {
818+
suffix = `REFs/${encodeURIComponent(REF)}`;
819+
}
820+
if (suffix) {
821+
await loadCommitsArray(`${baseUrl}/${suffix}/commits.json`);
822+
} else {
823+
console.error("PR or REF must be defined to load commits.");
866824
}
867825
const shaToLoad = (sha === 'latest') ? runtimeCache.commits[runtimeCache.commits.length - 1] : sha;
868826
if (runtimeCache.json0 === null) {
869-
await fetch0Json(PR, shaToLoad, nameParams[0]);
827+
await fetch0Json(suffix, shaToLoad, nameParams[0]);
870828
}
871829
if (nameParams.length > 1 && runtimeCache.json1 === null) {
872-
await fetch1Json(PR, shaToLoad, nameParams[1]);
830+
await fetch1Json(suffix, shaToLoad, nameParams[1]);
873831
}
874832
const footerRight = document.querySelector('#footer .right');
875833
function createLinkElement(href, textContent, footer) {
@@ -939,8 +897,8 @@
939897
async function init() {
940898
const urlParams = new URLSearchParams(window.location.search);
941899
const PR = urlParams.get('PR');
900+
const REF = urlParams.get('REF');
942901
const sha = urlParams.get('sha');
943-
const root_name = urlParams.get('name_0');
944902
const nameParams = [];
945903

946904
// Collect all `name_*` parameters into `nameParams` array
@@ -952,10 +910,16 @@
952910
});
953911

954912
// Render results using the extracted parameters
955-
await renderResults(PR, sha, nameParams);
913+
await renderResults(PR, REF, sha, nameParams);
956914
}
957915

958916
document.addEventListener('DOMContentLoaded', init);
917+
// document.querySelectorAll("a").forEach(link => {
918+
// link.addEventListener("click", function (event) {
919+
// event.preventDefault(); // Prevent full-page reload
920+
// history.pushState(null, "", this.href); // Update URL without reload
921+
// });
922+
// });
959923
</script>
960924
</body>
961925
</html>

ci/praktika/result.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,7 @@ def generate_skipped(cls, name, cache_record: Cache.CacheRecord, results=None):
279279
links=[
280280
Info().get_specific_report_url(
281281
pr_number=cache_record.pr_number,
282+
branch=cache_record.branch,
282283
sha=cache_record.sha,
283284
job_name=name,
284285
)

ci/praktika/s3.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,8 @@ def put(cls, s3_path, local_path, text=False, metadata=None, if_none_matched=Fal
8383
for k, v in metadata.items():
8484
command += f" --metadata {k}={v}"
8585

86-
cmd = f"aws s3 cp {local_path} s3://{s3_full_path}"
8786
if text:
88-
cmd += " --content-type text/plain"
87+
command += " --content-type text/plain"
8988
res = cls.run_command_with_retries(command)
9089
return res
9190

ci/praktika/settings.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import dataclasses
22
import importlib.util
33
from pathlib import Path
4-
from typing import Dict, Iterable, List, Optional, Union
4+
from typing import Dict, Iterable, List, Optional
55

66

77
@dataclasses.dataclass

0 commit comments

Comments
 (0)