Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions result_server/routes/admin.py
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,11 @@ def _build_execution_pipeline_plan(store):
result_server_url=_portal_result_server_url(),
target_id=gitlab_target.id if gitlab_target else gitlab_target_id,
)
variables = plan.payload.setdefault("variables", {})
manual_trigger_id = profile["id"] if profile else profile_id
variables["BK_TRIGGER_ID"] = manual_trigger_id
variables["BK_TRIGGER_TYPE"] = "manual_button"
variables["BK_TRIGGER_REASON"] = f"manual_button:{manual_trigger_id}"
request_errors = []
if not profile_id:
request_errors.append("profile_id is required")
Expand Down
3 changes: 2 additions & 1 deletion result_server/templates/_results_table.html
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@
.ci-pipeline {
display: block;
}
.ci-pipeline {
.ci-pipeline,
.ci-subline {
color: #607282;
font-size: 0.82em;
}
Expand Down
8 changes: 6 additions & 2 deletions result_server/templates/_results_table_cell_ci.html
Original file line number Diff line number Diff line change
@@ -1,4 +1,8 @@
<td class="ci-cell" title="{{ row.ci_trigger }} / {{ row.pipeline_id }}">
{% set pipeline_label = row.pipeline_label | default('#' ~ row.pipeline_id if row.pipeline_id != '-' else '-') %}
<td class="ci-cell" title="{{ row.ci_title | default(row.ci_trigger ~ ' / ' ~ pipeline_label) }}">
<span class="ci-trigger">{{ row.ci_trigger }}</span>
<span class="ci-pipeline">#{{ row.pipeline_id }}</span>
<span class="ci-pipeline">{{ pipeline_label }}</span>
{% if row.ci_subline | default('') %}
<span class="ci-subline">{{ row.ci_subline }}</span>
{% endif %}
</td>
6 changes: 6 additions & 0 deletions result_server/tests/test_execution_profiles.py
Original file line number Diff line number Diff line change
Expand Up @@ -1383,6 +1383,9 @@ def test_admin_execution_profiles_dry_run_submit_records_payload(tmp_path, monke
assert variables["code"] == "qws"
assert variables["BK_ALLOCATION_PROJECT_ID"] == "rkp00010"
assert variables["RESULT_SERVER"] == "http://localhost"
assert variables["BK_TRIGGER_ID"] == "rikyu-qws-nightly"
assert variables["BK_TRIGGER_TYPE"] == "manual_button"
assert variables["BK_TRIGGER_REASON"] == "manual_button:rikyu-qws-nightly"
assert "BK_SCHEDULER_EXTRA_ARGS_RIKYU" not in variables
assert "exp" not in variables
assert payload_record["gitlab_project"] == "gitlab.example.org/group/benchkit.git"
Expand Down Expand Up @@ -1755,6 +1758,9 @@ def fake_submit(plan, *, token):
assert plan.payload["variables"]["system"] == "Fugaku"
assert plan.payload["variables"]["BK_ALLOCATION_PROJECT_ID"] == "rkp00010"
assert plan.payload["variables"]["RESULT_SERVER"] == "http://localhost"
assert plan.payload["variables"]["BK_TRIGGER_ID"] == "rikyu-qws-nightly"
assert plan.payload["variables"]["BK_TRIGGER_TYPE"] == "manual_button"
assert plan.payload["variables"]["BK_TRIGGER_REASON"] == "manual_button:rikyu-qws-nightly"
assert "exp" not in plan.payload["variables"]
return GitLabPipelineSubmitResult(
status_code=201,
Expand Down
6 changes: 6 additions & 0 deletions result_server/tests/test_result_detail_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ def app():
"FOM_version": "osu-micro-benchmarks.osu_bibw.test_mpi_2",
"node_count": 1,
"cpus_per_node": 2,
"pipeline_id": 3208,
"parent_pipeline_id": 3207,
"metrics": {
"scalar": {"FOM": 6.47, "other_metric": 1.23},
"vector": {
Expand Down Expand Up @@ -100,6 +102,10 @@ def test_meta_info_section(self, app):
assert "6.470" in html
assert "MB/s" in html
assert "CPUs per Node" in html
assert "Pipeline ID" in html
assert "3208" in html
assert "Parent Pipeline ID" in html
assert "3207" in html
assert "Back to Results" in html
assert "Results" in html

Expand Down
9 changes: 9 additions & 0 deletions result_server/tests/test_results_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,6 +447,7 @@ def test_row_with_pipeline_timing_fields(self, flask_app, tmp_dir):
"build_job": "qws_Fugaku_build",
"run_job": "qws_Fugaku_N1_P4_T12_run",
"pipeline_id": 17026,
"parent_pipeline_id": 17025,
}
_write_json(tmp_dir, filename, data)

Expand All @@ -463,6 +464,10 @@ def test_row_with_pipeline_timing_fields(self, flask_app, tmp_dir):
assert row["build_job"] == "qws_Fugaku_build"
assert row["run_job"] == "qws_Fugaku_N1_P4_T12_run"
assert row["pipeline_id"] == "17026"
assert row["pipeline_label"] == "#17026"
assert row["parent_pipeline_id"] == "17025"
assert row["ci_title"] == "schedule / child #17026 / parent #17025"
assert row["ci_subline"] == "parent #17025"

def test_row_without_pipeline_timing_fields(self, flask_app, tmp_dir):
"""Test case."""
Expand All @@ -484,6 +489,10 @@ def test_row_without_pipeline_timing_fields(self, flask_app, tmp_dir):
assert row["build_job"] == "-"
assert row["run_job"] == "-"
assert row["pipeline_id"] == "-"
assert row["pipeline_label"] == "-"
assert row["parent_pipeline_id"] == ""
assert row["ci_title"] == "- / -"
assert row["ci_subline"] == ""

def test_row_with_partial_pipeline_timing(self, flask_app, tmp_dir):
"""Test case."""
Expand Down
46 changes: 46 additions & 0 deletions result_server/tests/test_trigger_display.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,28 @@ def test_summarize_execution_trigger_falls_back_to_pipeline_lookup():
assert summary["subline"] == "repo/ref changed: https://github.com/RIKEN-LQCD/qws.git@master"


def test_summarize_execution_trigger_falls_back_to_parent_pipeline_lookup():
runs = [
{
"trigger_id": "qws-fugaku-1400",
"trigger_type": "scheduled",
"status": "submitted",
"reason": "cron:0 14 * * *@2026-08-07T14:00+09:00",
"payload_json": {
"submit": {"response": {"id": 3189}},
},
}
]

summary = summarize_execution_trigger(
{"pipeline_id": 3190, "parent_pipeline_id": 3189},
build_trigger_run_lookup(runs),
)

assert summary["headline"] == "Scheduled / qws-fugaku-1400"
assert summary["subline"] == "cron 0 14 * * * / 2026-08-07T14:00+09:00"


def test_load_trigger_run_lookup_ignores_newer_routine_runs(tmp_path):
db_path = tmp_path / "cx_portal.sqlite3"
store = ExecutionProfileStore(str(db_path))
Expand Down Expand Up @@ -224,3 +246,27 @@ def test_build_trigger_result_links_matches_child_pipeline_fallback(tmp_path):

assert links[8][0]["filename"] == result_file.name
assert links[8][0]["pipeline_id"] == "3190"


def test_build_trigger_result_links_matches_parent_pipeline_fallback(tmp_path):
result_file = tmp_path / "result_20260807_140611_aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee.json"
result_file.write_text(
json.dumps({"code": "qws", "Exp": "CASE1", "pipeline_id": 3190, "parent_pipeline_id": 3189}),
encoding="utf-8",
)
runs = [
{
"id": 9,
"trigger_id": "qws-fugaku-1400",
"trigger_type": "scheduled",
"reason": "cron:0 14 * * *@2026-08-07T14:00+09:00",
"payload_json": {
"submit": {"response": {"id": 3189}},
},
}
]

links = build_trigger_result_links(str(tmp_path), runs)

assert links[9][0]["filename"] == result_file.name
assert links[9][0]["pipeline_id"] == "3190"
2 changes: 2 additions & 0 deletions result_server/utils/result_detail_view.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ def _build_meta_rows(result, trigger_runs_by_pipeline=None):
("FOM", format_numeric_value(result.get("FOM", "N/A"))),
("FOM Unit", result.get("FOM_unit") or "not specified"),
("Node Count", result.get("node_count", "N/A")),
("Pipeline ID", result.get("pipeline_id", "N/A")),
(
"Run Cause",
(
Expand All @@ -41,6 +42,7 @@ def _build_meta_rows(result, trigger_runs_by_pipeline=None):
("Processes per Node", result.get("numproc_node")),
("Threads per Process", result.get("nthreads")),
("CPUs per Node", result.get("cpus_per_node")),
("Parent Pipeline ID", result.get("parent_pipeline_id")),
]
for label, value in optional_rows:
if value not in (None, "", "N/A", "null"):
Expand Down
12 changes: 12 additions & 0 deletions result_server/utils/result_table_rows.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,14 @@ def build_result_table_row(
pipeline_id = "-"
else:
pipeline_id = str(pipeline_id)
pipeline_label = f"#{pipeline_id}" if pipeline_id != "-" else "-"
parent_pipeline_id = result_data.get("parent_pipeline_id", "")
parent_pipeline_id = str(parent_pipeline_id) if parent_pipeline_id not in (None, "") else ""
ci_title = f"{ci_trigger} / {pipeline_label}"
ci_subline = ""
if parent_pipeline_id and parent_pipeline_id != pipeline_id:
ci_title = f"{ci_trigger} / child #{pipeline_id} / parent #{parent_pipeline_id}"
ci_subline = f"parent #{parent_pipeline_id}"

return {
"timestamp": timestamp,
Expand All @@ -54,13 +62,17 @@ def build_result_table_row(
"execution_mode": result_data.get("execution_mode", "-") or "-",
"ci_trigger": ci_trigger,
"ci_summary": f"{ci_trigger} / {pipeline_id}",
"ci_title": ci_title,
"ci_subline": ci_subline,
"pipeline_label": pipeline_label,
"execution_trigger_summary": summarize_execution_trigger(
result_data,
trigger_runs_by_pipeline,
),
"build_job": result_data.get("build_job", "-") or "-",
"run_job": result_data.get("run_job", "-") or "-",
"pipeline_id": pipeline_id,
"parent_pipeline_id": parent_pipeline_id,
"source_info": source_info,
"source_link": source_link,
"source_hash": _format_source_hash(source_info),
Expand Down
20 changes: 16 additions & 4 deletions result_server/utils/trigger_display.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,8 @@ def _index_trigger_run(run: dict[str, Any]) -> dict[str, Any]:


def _result_matches_trigger_run(result: dict[str, Any], run: dict[str, Any]) -> bool:
pipeline_id = str(result.get("pipeline_id") or "").strip()
if pipeline_id and pipeline_id in run["pipeline_ids"]:
result_pipeline_ids = _result_pipeline_ids(result)
if set(result_pipeline_ids).intersection(run["pipeline_ids"]):
return True
trigger = extract_execution_trigger(result)
if not trigger["id"]:
Expand Down Expand Up @@ -205,8 +205,11 @@ def _execution_trigger_from_pipeline(
trigger_runs_by_pipeline: dict[str, dict[str, Any]],
) -> dict[str, str]:
data = result if isinstance(result, dict) else {}
pipeline_id = str(data.get("pipeline_id") or "").strip()
run = trigger_runs_by_pipeline.get(pipeline_id) if pipeline_id else None
run = None
for pipeline_id in _result_pipeline_ids(data):
run = trigger_runs_by_pipeline.get(pipeline_id)
if run:
break
if not run:
return {"id": "", "type": "", "reason": ""}
return {
Expand All @@ -231,6 +234,15 @@ def _trigger_run_pipeline_ids(run: dict[str, Any]) -> list[str]:
]


def _result_pipeline_ids(result: dict[str, Any]) -> list[str]:
values = []
for value in (result.get("pipeline_id"), result.get("parent_pipeline_id")):
text = str(value).strip() if value not in (None, "") else ""
if text and text not in values:
values.append(text)
return values


def _format_trigger_type(trigger_type: str) -> str:
mapping = {
"manual_button": "Manual",
Expand Down
2 changes: 2 additions & 0 deletions scripts/matrix_generate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ QUEUE_FILE="config/queue.csv"
SYSTEM_INFO_FILE="config/system_info.csv"
OUTPUT_FILE=".gitlab-ci.generated.yml"
PARENT_PIPELINE_SOURCE="${CI_PIPELINE_SOURCE:-local}"
PARENT_PIPELINE_ID="${CI_PIPELINE_ID:-}"

source ./scripts/job_functions.sh

Expand Down Expand Up @@ -43,6 +44,7 @@ stages:

variables:
PARENT_PIPELINE_SOURCE: \"$PARENT_PIPELINE_SOURCE\"
PARENT_PIPELINE_ID: \"$PARENT_PIPELINE_ID\"
BK_ESTIMATE_RUNNER_TAG: \"$ESTIMATE_RUNNER_TAG\"
" >> "$OUTPUT_FILE"

Expand Down
8 changes: 7 additions & 1 deletion scripts/result.sh
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,12 @@ write_result_json() {
\"pipeline_id\": $pipeline_id"
fi

local parent_pipeline_id_block=""
if [ -n "${PARENT_PIPELINE_ID:-}" ]; then
parent_pipeline_id_block=",
\"parent_pipeline_id\": ${PARENT_PIPELINE_ID}"
fi

local execution_trigger_block=""
if [ -n "${BK_TRIGGER_ID:-}" ] || [ -n "${BK_TRIGGER_TYPE:-}" ] || [ -n "${BK_TRIGGER_REASON:-}" ]; then
local execution_trigger_json
Expand Down Expand Up @@ -272,7 +278,7 @@ write_result_json() {
"nthreads": "$nthreads",
"description": "$description",
"confidential": "$confidential",
"source_info": $source_info_block${profile_data_block}${fom_breakdown_block}${timing_block}${mode_block}${trigger_block}${build_job_block}${run_job_block}${pipeline_id_block}${execution_trigger_block}
"source_info": $source_info_block${profile_data_block}${fom_breakdown_block}${timing_block}${mode_block}${trigger_block}${build_job_block}${run_job_block}${pipeline_id_block}${parent_pipeline_id_block}${execution_trigger_block}
}
EOF

Expand Down
3 changes: 3 additions & 0 deletions scripts/tests/test_process_and_send_results.sh
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ export RESULT_SERVER_CLIENT_KEY="${TMP_DIR}/client.key"
export BK_TRIGGER_ID="qws-fugaku-1400"
export BK_TRIGGER_TYPE="scheduled"
export BK_TRIGGER_REASON="cron:0 14 * * *@2026-08-07T14:00+09:00"
export PARENT_PIPELINE_ID="54321"

pushd "${TMP_DIR}/project" >/dev/null
bash scripts/result_server/process_and_send_results.sh qws Fugaku cross qws_Fugaku_build qws_Fugaku_N1_P2_T3_run 12345
Expand All @@ -62,6 +63,8 @@ jq -e '._server_uuid == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"' \
"${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
jq -e '.execution_trigger.id == "qws-fugaku-1400" and .execution_trigger.type == "scheduled"' \
"${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
jq -e '.pipeline_id == 12345 and .parent_pipeline_id == 54321' \
"${TMP_DIR}/project/send_results_workspace/results/result0.json" >/dev/null
jq -e '."result0.json".uuid == "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee"' \
"${TMP_DIR}/project/send_results_workspace/results/server_result_meta.json" >/dev/null

Expand Down
5 changes: 4 additions & 1 deletion scripts/tests/test_result_profile_data.sh
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,9 @@ pushd "${TMP_DIR}" >/dev/null
export BK_TRIGGER_ID="qws-fugaku-watch"
export BK_TRIGGER_TYPE="watch_event"
export BK_TRIGGER_REASON="repo_ref:https://github.com/RIKEN-LQCD/qws.git@master"
export PARENT_PIPELINE_ID="888"
bash "${REPO_DIR}/scripts/result.sh" qws Fugaku cross build run 999 >/dev/null
unset BK_TRIGGER_ID BK_TRIGGER_TYPE BK_TRIGGER_REASON
unset BK_TRIGGER_ID BK_TRIGGER_TYPE BK_TRIGGER_REASON PARENT_PIPELINE_ID
popd >/dev/null

pushd "${TMP_DIR}/ncu" >/dev/null
Expand All @@ -114,6 +115,8 @@ jq -e '
.pipeline_timing.build_time == 12 and
.pipeline_timing.queue_time == 0 and
.pipeline_timing.run_time == 34 and
.pipeline_id == 999 and
.parent_pipeline_id == 888 and
.execution_trigger.id == "qws-fugaku-watch" and
.execution_trigger.type == "watch_event" and
.execution_trigger.reason == "repo_ref:https://github.com/RIKEN-LQCD/qws.git@master" and
Expand Down
Loading