From 8a5164fa0e4ba8c27719a4f2ba92f2512a89f508 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Sun, 21 Jun 2026 20:32:55 +0300 Subject: [PATCH 1/6] Fix: Maestro validation retry counting Retry filtering only happened during missing-id checks. Counts and boot status checks still used raw Maestro nodes. Share the filtered Maestro set and keep missing checks dashboard-only. Add regression tests for count, status, and missing-id behavior. Signed-off-by: Denys Fedoryshchenko --- kcidev/subcommands/maestro/validate/helper.py | 46 +++++++++-------- tests/test_maestro_validate_json.py | 51 ++++++++++++++++++- 2 files changed, 76 insertions(+), 21 deletions(-) diff --git a/kcidev/subcommands/maestro/validate/helper.py b/kcidev/subcommands/maestro/validate/helper.py index 9cf5bb9..f7f726b 100644 --- a/kcidev/subcommands/maestro/validate/helper.py +++ b/kcidev/subcommands/maestro/validate/helper.py @@ -57,37 +57,40 @@ def get_builds(ctx, giturl, branch, commit, arch, raise_errors=False): return maestro_builds, dashboard_builds -def find_missing_items(maestro_data, dashboard_data, item_type, verbose): - """ - Compare build/boot IDs found in maestro with dashboard results - Return missing builds/boots in dashboard. +def filter_maestro_items_for_validation(maestro_data, item_type): """ - dashboard_ids = [b["id"].split(":")[1] for b in dashboard_data] + Return Maestro items that should be compared against dashboard results. + Retry filtering must be shared by counts, missing-item checks, and status + checks so all validation fields describe the same item set. + """ if item_type == "build": # Exclude build retries - maestro_data = [ + return [ b for b in maestro_data if (b["result"] != "incomplete" or b["retry_counter"] == 3) ] - elif item_type == "boot": + if item_type == "boot": # Exclude boot retries - maestro_data = [ + return [ b for b in maestro_data if (b["result"] not in {"incomplete", "fail"} or b["retry_counter"] == 3) ] + return maestro_data - maestro_ids = [b["id"] for b in maestro_data] - if len(maestro_ids) > len(dashboard_ids): - missing_items = [b for b in maestro_data if b["id"] not in dashboard_ids] - missing_ids = [b["id"] for b in missing_items] - else: - missing_items = [ - b for b in dashboard_data if b["id"].split(":")[1] not in maestro_ids - ] - missing_ids = [b["id"].split(":")[1] for b in missing_items] + +def find_missing_items(maestro_data, dashboard_data, item_type, verbose): + """ + Compare build/boot IDs found in maestro with dashboard results + Return missing builds/boots in dashboard. + """ + maestro_data = filter_maestro_items_for_validation(maestro_data, item_type) + dashboard_ids = [b["id"].split(":")[1] for b in dashboard_data] + + missing_items = [b for b in maestro_data if b["id"] not in dashboard_ids] + missing_ids = [b["id"] for b in missing_items] if missing_items and verbose: kci_msg("Missing items:") @@ -127,6 +130,7 @@ def get_build_stats( ) if dashboard_builds is None: return [] + maestro_builds = filter_maestro_items_for_validation(maestro_builds, "build") summary_flag = "✅" missing_build_ids = find_missing_items( maestro_builds, dashboard_builds, "build", verbose @@ -402,6 +406,7 @@ def get_boot_stats( ) if dashboard_boots is None: return [] + maestro_boots = filter_maestro_items_for_validation(maestro_boots, "boot") summary_flag = "✅" missing_boot_ids = find_missing_items( maestro_boots, dashboard_boots, "boot", verbose @@ -432,10 +437,11 @@ def get_builds_history_stats( final_stats = [] builds_history = get_builds_history(ctx, checkouts, arch, raise_errors) for b in builds_history: - missing_build_ids = find_missing_items(b[1], b[2], "build", verbose) - total_maestro_builds = len(b[1]) + maestro_builds = filter_maestro_items_for_validation(b[1], "build") + missing_build_ids = find_missing_items(maestro_builds, b[2], "build", verbose) + total_maestro_builds = len(maestro_builds) total_dashboard_builds = len(b[2]) - mismatched_ids = validate_build_status(b[1], b[2]) + mismatched_ids = validate_build_status(maestro_builds, b[2]) summary_flag = ( "✅" if total_maestro_builds == total_dashboard_builds diff --git a/tests/test_maestro_validate_json.py b/tests/test_maestro_validate_json.py index e70e4cc..b9a4f3d 100644 --- a/tests/test_maestro_validate_json.py +++ b/tests/test_maestro_validate_json.py @@ -4,7 +4,10 @@ import click from click.testing import CliRunner -from kcidev.subcommands.maestro.validate.helper import validation_rows_to_json +from kcidev.subcommands.maestro.validate.helper import ( + find_missing_items, + validation_rows_to_json, +) COMMIT = "a" * 40 @@ -351,3 +354,49 @@ def test_mismatch_without_fail_on_mismatch_still_exits_zero(monkeypatch): result = CliRunner().invoke(command, args) assert result.exit_code == 0 + + +def test_missing_items_only_reports_maestro_items_missing_from_dashboard(): + maestro_data = [{"id": "boot-1", "result": "pass", "retry_counter": 0}] + dashboard_data = [ + {"id": "maestro:boot-1", "status": "PASS"}, + {"id": "maestro:dashboard-extra", "status": "PASS"}, + ] + + assert find_missing_items(maestro_data, dashboard_data, "boot", False) == [] + + +def test_boot_stats_uses_retry_filtered_maestro_count_and_status(monkeypatch): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + maestro_boots = [ + {"id": "boot-1", "result": "pass", "retry_counter": 0, "data": {}}, + {"id": "boot-retry", "result": "fail", "retry_counter": 1, "data": {}}, + ] + dashboard_boots = [ + {"id": "maestro:boot-1", "status": "PASS"}, + {"id": "maestro:boot-retry", "status": "PASS"}, + ] + monkeypatch.setattr( + helper_module, + "get_boots", + lambda ctx, giturl, branch, commit, arch, raise_errors=False: ( + maestro_boots, + dashboard_boots, + ), + ) + + stats = helper_module.get_boot_stats( + None, + "https://example.com/linux.git", + "master", + COMMIT, + "mainline", + False, + None, + ) + + assert stats[2] == 1 + assert stats[3] == 2 + assert stats[6] == [] From a74e4b40f43e5c6c7efb99d802b2db0979dd6cf7 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Sun, 21 Jun 2026 20:49:45 +0300 Subject: [PATCH 2/6] Add human-readable validation output Signed-off-by: Denys Fedoryshchenko --- kcidev/subcommands/maestro/validate/boots.py | 15 ++++ kcidev/subcommands/maestro/validate/builds.py | 15 ++++ kcidev/subcommands/maestro/validate/helper.py | 77 +++++++++++++++++++ tests/test_maestro_validate_json.py | 65 ++++++++++++++++ 4 files changed, 172 insertions(+) diff --git a/kcidev/subcommands/maestro/validate/boots.py b/kcidev/subcommands/maestro/validate/boots.py index 7c28586..9400a0c 100644 --- a/kcidev/subcommands/maestro/validate/boots.py +++ b/kcidev/subcommands/maestro/validate/boots.py @@ -9,6 +9,7 @@ get_boot_stats, print_simple_list, print_table_stats, + print_validation_human, print_validation_json, validation_rows_to_json, ) @@ -90,6 +91,12 @@ default=False, help="Print validation results as JSON", ) +@click.option( + "--human-readable", + is_flag=True, + default=False, + help="Print a human-readable validation summary. With --json, text is written to stderr.", +) @click.option( "--fail-on-mismatch", is_flag=True, @@ -111,6 +118,7 @@ def boots( verbose, table_output, use_json, + human_readable, fail_on_mismatch, ): final_stats = [] @@ -165,6 +173,8 @@ def boots( sys.stdout = stdout if use_json: report = print_validation_json("boots", final_stats, False, origin, days, arch) + if human_readable: + print_validation_human(report, err=True) if fail_on_mismatch and not report["summary"]["ok"]: ctx.exit(1) return @@ -182,6 +192,11 @@ def boots( table_fmt = "simple_grid" if table_output: print_table_stats(final_stats, headers, max_col_width, table_fmt) + elif human_readable: + report = validation_rows_to_json( + "boots", final_stats, False, origin, days, arch + ) + print_validation_human(report) else: print_simple_list(final_stats, "boots", False) if fail_on_mismatch: diff --git a/kcidev/subcommands/maestro/validate/builds.py b/kcidev/subcommands/maestro/validate/builds.py index 248bb4c..9e089a1 100644 --- a/kcidev/subcommands/maestro/validate/builds.py +++ b/kcidev/subcommands/maestro/validate/builds.py @@ -10,6 +10,7 @@ get_builds_history_stats, print_simple_list, print_table_stats, + print_validation_human, print_validation_json, validation_rows_to_json, ) @@ -104,6 +105,12 @@ default=False, help="Print validation results as JSON", ) +@click.option( + "--human-readable", + is_flag=True, + default=False, + help="Print a human-readable validation summary. With --json, text is written to stderr.", +) @click.option( "--fail-on-mismatch", is_flag=True, @@ -126,6 +133,7 @@ def builds( verbose, table_output, use_json, + human_readable, fail_on_mismatch, ): final_stats = [] @@ -213,6 +221,8 @@ def builds( report = print_validation_json( "builds", final_stats, history, origin, days, arch ) + if human_readable: + print_validation_human(report, err=True) if fail_on_mismatch and not report["summary"]["ok"]: ctx.exit(1) return @@ -242,6 +252,11 @@ def builds( table_fmt = "simple_grid" if table_output: print_table_stats(final_stats, headers, max_col_width, table_fmt) + elif human_readable: + report = validation_rows_to_json( + "builds", final_stats, history, origin, days, arch + ) + print_validation_human(report) else: print_simple_list(final_stats, "builds", history) if fail_on_mismatch: diff --git a/kcidev/subcommands/maestro/validate/helper.py b/kcidev/subcommands/maestro/validate/helper.py index f7f726b..a361fa3 100644 --- a/kcidev/subcommands/maestro/validate/helper.py +++ b/kcidev/subcommands/maestro/validate/helper.py @@ -293,6 +293,83 @@ def validation_rows_to_json(kind, rows, history, origin, days, arch): } +def validation_report_to_text(report): + """Render a JSON validation report as human-readable text.""" + item_type = report["kind"][:-1] if report["kind"].endswith("s") else report["kind"] + summary = report["summary"] + scope = [ + f"origin={report['origin']}", + f"days={report['days']}", + f"history={'yes' if report['history'] else 'no'}", + ] + if report.get("arch"): + scope.append(f"arch={report['arch']}") + else: + scope.append("arch=all") + + status = "PASS" if summary["ok"] else "FAIL" + lines = [ + f"Dashboard validation: {report['kind']} ({status})", + f"Scope: {', '.join(scope)}", + ( + "Summary: " + f"checked={summary['checked']}, " + f"failed={summary['failed']}, " + f"missing={summary['missing']}, " + f"status_mismatches={summary['status_mismatches']}" + ), + ] + + if report["results"]: + lines.append("") + lines.append( + "Note: count comparisons use raw query totals; missing IDs are " + "reported after validation filtering." + ) + + for result in report["results"]: + result_status = "PASS" if result["ok"] else "FAIL" + commit = result["commit"] + delta = result["maestro_count"] - result["dashboard_count"] + if delta > 0: + delta_text = f"{delta} more in Maestro" + elif delta < 0: + delta_text = f"{-delta} more in dashboard" + else: + delta_text = "counts match" + + lines.extend( + [ + "", + f"{result['tree_branch']} @ {commit[:12]}: {result_status}", + f" Commit: {commit}", + f" Maestro {report['kind']}: {result['maestro_count']}", + f" Dashboard {report['kind']}: {result['dashboard_count']}", + f" Count delta: {delta_text}", + ] + ) + + missing_ids = result["missing_ids"] + status_mismatch_ids = result["status_mismatch_ids"] + if missing_ids: + lines.append(f" Missing {item_type} IDs: {len(missing_ids)}") + for node_id in missing_ids: + lines.append(f" - https://api.kernelci.org/viewer?node_id={node_id}") + if status_mismatch_ids: + lines.append( + f" Status mismatch {item_type} IDs: {len(status_mismatch_ids)}" + ) + for node_id in status_mismatch_ids: + lines.append(f" - https://api.kernelci.org/viewer?node_id={node_id}") + + return "\n".join(lines) + + +def print_validation_human(report, err=False): + """Print a human-readable validation report.""" + click.echo(validation_report_to_text(report), err=err) + + def print_validation_json(kind, rows, history, origin, days, arch): """Print a single JSON validation report to stdout.""" report = validation_rows_to_json(kind, rows, history, origin, days, arch) diff --git a/tests/test_maestro_validate_json.py b/tests/test_maestro_validate_json.py index b9a4f3d..3828d95 100644 --- a/tests/test_maestro_validate_json.py +++ b/tests/test_maestro_validate_json.py @@ -6,6 +6,7 @@ from kcidev.subcommands.maestro.validate.helper import ( find_missing_items, + validation_report_to_text, validation_rows_to_json, ) @@ -16,6 +17,10 @@ def _stdout(result): return getattr(result, "stdout", result.output) +def _stderr(result): + return getattr(result, "stderr", "") + + def _build_row( maestro_count=1, dashboard_count=1, @@ -221,6 +226,66 @@ def test_json_mismatches_still_exit_zero(monkeypatch): assert json.loads(_stdout(result))["summary"]["ok"] is False +def test_validation_report_to_text_decodes_summary(): + report = validation_rows_to_json( + "boots", + [ + _build_row( + 116, + 45, + "❌", + missing_ids=["boot-1"], + status_mismatch_ids=["boot-2"], + ) + ], + False, + "maestro", + 1, + None, + ) + + text = validation_report_to_text(report) + + assert "Dashboard validation: boots (FAIL)" in text + assert "Summary: checked=1, failed=1, missing=1, status_mismatches=1" in text + assert "mainline/master @ aaaaaaaaaaaa: FAIL" in text + assert "Count delta: 71 more in Maestro" in text + assert "https://api.kernelci.org/viewer?node_id=boot-1" in text + assert "https://api.kernelci.org/viewer?node_id=boot-2" in text + + +def test_json_human_readable_keeps_stdout_json_and_writes_text_to_stderr( + monkeypatch, +): + command = _patch_boots_command( + monkeypatch, + _build_row(2, 1, "❌", missing_ids=["boot-1"]), + ) + args = _json_args() + ["--human-readable"] + + result = CliRunner().invoke(command, args) + + assert result.exit_code == 0 + report = json.loads(_stdout(result)) + assert report["summary"]["ok"] is False + assert "Dashboard validation: boots (FAIL)" in _stderr(result) + assert "node_id=boot-1" in _stderr(result) + + +def test_human_readable_without_json_uses_decoded_text(monkeypatch): + command = _patch_builds_command( + monkeypatch, + _build_row(2, 1, "❌", missing_ids=["build-1"]), + ) + args = _json_args()[:-1] + ["--human-readable"] + + result = CliRunner().invoke(command, args) + + assert result.exit_code == 0 + assert "Dashboard validation: builds (FAIL)" in _stdout(result) + assert "node_id=build-1" in _stdout(result) + + def test_json_runtime_failures_exit_nonzero(monkeypatch): builds_module = importlib.import_module( "kcidev.subcommands.maestro.validate.builds" From b55fdc4ea9432bc51e9094d7fd6339e2e3fce1be Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Sun, 21 Jun 2026 21:44:19 +0300 Subject: [PATCH 3/6] Report dashboard-extra validation IDs Signed-off-by: Denys Fedoryshchenko --- kcidev/subcommands/maestro/validate/boots.py | 3 +- kcidev/subcommands/maestro/validate/builds.py | 6 +- kcidev/subcommands/maestro/validate/helper.py | 77 ++++++++++++++++++- tests/test_maestro_validate_json.py | 55 ++++++++++++- tests/test_maestro_validate_stats.py | 3 +- 5 files changed, 135 insertions(+), 9 deletions(-) diff --git a/kcidev/subcommands/maestro/validate/boots.py b/kcidev/subcommands/maestro/validate/boots.py index 9400a0c..b346f76 100644 --- a/kcidev/subcommands/maestro/validate/boots.py +++ b/kcidev/subcommands/maestro/validate/boots.py @@ -187,8 +187,9 @@ def boots( "Boot count\ncomparison", "Missing boot IDs", "Boots with\nstatus mismatch", + "Extra dashboard\nboot IDs", ] - max_col_width = [None, 40, 3, 3, 2, 30, 30] + max_col_width = [None, 40, 3, 3, 2, 30, 30, 30] table_fmt = "simple_grid" if table_output: print_table_stats(final_stats, headers, max_col_width, table_fmt) diff --git a/kcidev/subcommands/maestro/validate/builds.py b/kcidev/subcommands/maestro/validate/builds.py index 9e089a1..beee270 100644 --- a/kcidev/subcommands/maestro/validate/builds.py +++ b/kcidev/subcommands/maestro/validate/builds.py @@ -235,8 +235,9 @@ def builds( "Dashboard\nbuilds", "Build count\ncomparison", "Missing build IDs", + "Extra dashboard\nbuild IDs", ] - max_col_width = [None, 40, 3, 3, 2, 30] + max_col_width = [None, 40, 3, 3, 2, 30, 30] else: headers = [ @@ -247,8 +248,9 @@ def builds( "Build count\ncomparison", "Missing build IDs", "Builds with\nstatus mismatch", + "Extra dashboard\nbuild IDs", ] - max_col_width = [None, 40, 3, 3, 2, 30, 30] + max_col_width = [None, 40, 3, 3, 2, 30, 30, 30] table_fmt = "simple_grid" if table_output: print_table_stats(final_stats, headers, max_col_width, table_fmt) diff --git a/kcidev/subcommands/maestro/validate/helper.py b/kcidev/subcommands/maestro/validate/helper.py index a361fa3..cca75e4 100644 --- a/kcidev/subcommands/maestro/validate/helper.py +++ b/kcidev/subcommands/maestro/validate/helper.py @@ -99,6 +99,26 @@ def find_missing_items(maestro_data, dashboard_data, item_type, verbose): return missing_ids +def find_extra_dashboard_items(maestro_data, dashboard_data, item_type, verbose): + """ + Compare build/boot IDs found in dashboard with Maestro results. + Return dashboard builds/boots that are not present in Maestro. + """ + maestro_data = filter_maestro_items_for_validation(maestro_data, item_type) + maestro_ids = {b["id"] for b in maestro_data} + + extra_items = [ + b for b in dashboard_data if b["id"].split(":")[1] not in maestro_ids + ] + extra_ids = [b["id"].split(":")[1] for b in extra_items] + + if extra_items and verbose: + kci_msg("Extra dashboard items:") + kci_msg_json(extra_items) + + return extra_ids + + def validate_build_status(maestro_builds, dashboard_builds): """ Validate if the build status of dashboard pulled build @@ -135,13 +155,16 @@ def get_build_stats( missing_build_ids = find_missing_items( maestro_builds, dashboard_builds, "build", verbose ) - + extra_dashboard_build_ids = find_extra_dashboard_items( + maestro_builds, dashboard_builds, "build", verbose + ) builds_with_status_mismatch = validate_build_status( maestro_builds, dashboard_builds ) if ( len(dashboard_builds) != len(maestro_builds) or missing_build_ids + or extra_dashboard_build_ids or builds_with_status_mismatch ): summary_flag = "❌" @@ -153,6 +176,7 @@ def get_build_stats( summary_flag, missing_build_ids, builds_with_status_mismatch, + extra_dashboard_build_ids, ] return stats @@ -173,7 +197,15 @@ def extract_validation_data(row: list): comparison = row[4] # Build/boot count comparison (✅ or ❌) missing_ids = row[5] # Missing build/boot IDs mismatched_ids = row[6] - return tree_branch, commit, comparison, missing_ids, mismatched_ids + extra_dashboard_ids = row[7] or [] if len(row) > 7 else [] + return ( + tree_branch, + commit, + comparison, + missing_ids, + mismatched_ids, + extra_dashboard_ids, + ) except IndexError: kci_msg_red("Failed to extract data for list view report") raise ValueError() @@ -196,6 +228,7 @@ def print_simple_list(data, item_type, history=False): comparison, missing_ids, mismatched_ids, + extra_dashboard_ids, ) = extract_validation_data(row) except ValueError: continue @@ -205,6 +238,7 @@ def print_simple_list(data, item_type, history=False): "status": comparison, "missing_ids": missing_ids, "mismatched_ids": mismatched_ids, + "extra_dashboard_ids": extra_dashboard_ids, } ) @@ -223,6 +257,12 @@ def print_simple_list(data, item_type, history=False): ) for id in commit["mismatched_ids"]: kci_msg(f" - https://api.kernelci.org/viewer?node_id={id}") + if commit["extra_dashboard_ids"]: + kci_msg( + f" Extra dashboard {item_type}: {len(commit['extra_dashboard_ids'])}" + ) + for id in commit["extra_dashboard_ids"]: + kci_msg(f" - https://api.kernelci.org/viewer?node_id={id}") elif commit["status"] == "✅": kci_msg(f" Commit {commit['commit'][:12]}: ✅") @@ -236,6 +276,7 @@ def print_simple_list(data, item_type, history=False): comparison, missing_ids, mismatched_ids, + extra_dashboard_ids, ) = extract_validation_data(row) except ValueError: continue @@ -251,6 +292,10 @@ def print_simple_list(data, item_type, history=False): kci_msg(f" Status mismatched {item_type}: {len(mismatched_ids)}") for id in mismatched_ids: kci_msg(f" - https://api.kernelci.org/viewer?node_id={id}") + if comparison == "❌" and extra_dashboard_ids: + kci_msg(f" Extra dashboard {item_type}: {len(extra_dashboard_ids)}") + for id in extra_dashboard_ids: + kci_msg(f" - https://api.kernelci.org/viewer?node_id={id}") kci_msg("") @@ -260,8 +305,9 @@ def validation_rows_to_json(kind, rows, history, origin, days, arch): for row in rows: missing_ids = row[5] or [] status_mismatch_ids = row[6] or [] + extra_dashboard_ids = row[7] or [] if len(row) > 7 else [] ok = row[2] == row[3] and row[4] == "✅" and not missing_ids - ok = ok and not status_mismatch_ids + ok = ok and not status_mismatch_ids and not extra_dashboard_ids results.append( { "tree_branch": row[0], @@ -270,6 +316,7 @@ def validation_rows_to_json(kind, rows, history, origin, days, arch): "dashboard_count": row[3], "ok": ok, "missing_ids": missing_ids, + "extra_in_dashboard_ids": extra_dashboard_ids, "status_mismatch_ids": status_mismatch_ids, } ) @@ -284,6 +331,9 @@ def validation_rows_to_json(kind, rows, history, origin, days, arch): "checked": len(results), "failed": sum(not result["ok"] for result in results), "missing": sum(len(result["missing_ids"]) for result in results), + "extra_in_dashboard": sum( + len(result["extra_in_dashboard_ids"]) for result in results + ), "status_mismatches": sum( len(result["status_mismatch_ids"]) for result in results ), @@ -316,6 +366,7 @@ def validation_report_to_text(report): f"checked={summary['checked']}, " f"failed={summary['failed']}, " f"missing={summary['missing']}, " + f"extra_in_dashboard={summary['extra_in_dashboard']}, " f"status_mismatches={summary['status_mismatches']}" ), ] @@ -324,7 +375,8 @@ def validation_report_to_text(report): lines.append("") lines.append( "Note: count comparisons use raw query totals; missing IDs are " - "reported after validation filtering." + "reported after validation filtering. Extra dashboard IDs are " + "dashboard rows not present in the filtered Maestro set." ) for result in report["results"]: @@ -350,11 +402,18 @@ def validation_report_to_text(report): ) missing_ids = result["missing_ids"] + extra_dashboard_ids = result["extra_in_dashboard_ids"] status_mismatch_ids = result["status_mismatch_ids"] if missing_ids: lines.append(f" Missing {item_type} IDs: {len(missing_ids)}") for node_id in missing_ids: lines.append(f" - https://api.kernelci.org/viewer?node_id={node_id}") + if extra_dashboard_ids: + lines.append( + f" Extra dashboard {item_type} IDs: {len(extra_dashboard_ids)}" + ) + for node_id in extra_dashboard_ids: + lines.append(f" - https://api.kernelci.org/viewer?node_id={node_id}") if status_mismatch_ids: lines.append( f" Status mismatch {item_type} IDs: {len(status_mismatch_ids)}" @@ -488,10 +547,14 @@ def get_boot_stats( missing_boot_ids = find_missing_items( maestro_boots, dashboard_boots, "boot", verbose ) + extra_dashboard_boot_ids = find_extra_dashboard_items( + maestro_boots, dashboard_boots, "boot", verbose + ) boots_with_status_mismatch = validate_boot_status(maestro_boots, dashboard_boots) if ( len(dashboard_boots) != len(maestro_boots) or missing_boot_ids + or extra_dashboard_boot_ids or boots_with_status_mismatch ): summary_flag = "❌" @@ -503,6 +566,7 @@ def get_boot_stats( summary_flag, missing_boot_ids, boots_with_status_mismatch, + extra_dashboard_boot_ids, ] return stats @@ -516,6 +580,9 @@ def get_builds_history_stats( for b in builds_history: maestro_builds = filter_maestro_items_for_validation(b[1], "build") missing_build_ids = find_missing_items(maestro_builds, b[2], "build", verbose) + extra_dashboard_build_ids = find_extra_dashboard_items( + maestro_builds, b[2], "build", verbose + ) total_maestro_builds = len(maestro_builds) total_dashboard_builds = len(b[2]) mismatched_ids = validate_build_status(maestro_builds, b[2]) @@ -523,6 +590,7 @@ def get_builds_history_stats( "✅" if total_maestro_builds == total_dashboard_builds and not missing_build_ids + and not extra_dashboard_build_ids and not mismatched_ids else "❌" ) @@ -534,6 +602,7 @@ def get_builds_history_stats( summary_flag, missing_build_ids, mismatched_ids, + extra_dashboard_build_ids, ] final_stats.append(stats) return final_stats diff --git a/tests/test_maestro_validate_json.py b/tests/test_maestro_validate_json.py index 3828d95..fdcdc31 100644 --- a/tests/test_maestro_validate_json.py +++ b/tests/test_maestro_validate_json.py @@ -5,6 +5,7 @@ from click.testing import CliRunner from kcidev.subcommands.maestro.validate.helper import ( + find_extra_dashboard_items, find_missing_items, validation_report_to_text, validation_rows_to_json, @@ -27,6 +28,7 @@ def _build_row( summary_flag="✅", missing_ids=None, status_mismatch_ids=None, + extra_dashboard_ids=None, ): return [ "mainline/master", @@ -36,6 +38,7 @@ def _build_row( summary_flag, missing_ids or [], status_mismatch_ids or [], + extra_dashboard_ids or [], ] @@ -184,6 +187,23 @@ def test_missing_ids_make_summary_not_ok(): assert report["results"][0]["ok"] is False +def test_extra_dashboard_ids_make_summary_not_ok(): + report = validation_rows_to_json( + "boots", + [_build_row(1, 2, "❌", extra_dashboard_ids=["boot-extra"])], + False, + "maestro", + 1, + None, + ) + + assert report["summary"]["ok"] is False + assert report["summary"]["failed"] == 1 + assert report["summary"]["extra_in_dashboard"] == 1 + assert report["results"][0]["extra_in_dashboard_ids"] == ["boot-extra"] + assert report["results"][0]["ok"] is False + + def test_status_mismatches_make_summary_not_ok(): report = validation_rows_to_json( "boots", @@ -247,13 +267,33 @@ def test_validation_report_to_text_decodes_summary(): text = validation_report_to_text(report) assert "Dashboard validation: boots (FAIL)" in text - assert "Summary: checked=1, failed=1, missing=1, status_mismatches=1" in text + assert ( + "Summary: checked=1, failed=1, missing=1, extra_in_dashboard=0, " + "status_mismatches=1" + ) in text assert "mainline/master @ aaaaaaaaaaaa: FAIL" in text assert "Count delta: 71 more in Maestro" in text assert "https://api.kernelci.org/viewer?node_id=boot-1" in text assert "https://api.kernelci.org/viewer?node_id=boot-2" in text +def test_validation_report_to_text_shows_extra_dashboard_ids(): + report = validation_rows_to_json( + "boots", + [_build_row(1, 2, "❌", extra_dashboard_ids=["boot-extra"])], + False, + "maestro", + 1, + None, + ) + + text = validation_report_to_text(report) + + assert "Count delta: 1 more in dashboard" in text + assert "Extra dashboard boot IDs: 1" in text + assert "https://api.kernelci.org/viewer?node_id=boot-extra" in text + + def test_json_human_readable_keeps_stdout_json_and_writes_text_to_stderr( monkeypatch, ): @@ -431,6 +471,18 @@ def test_missing_items_only_reports_maestro_items_missing_from_dashboard(): assert find_missing_items(maestro_data, dashboard_data, "boot", False) == [] +def test_extra_dashboard_items_report_dashboard_items_missing_from_maestro(): + maestro_data = [{"id": "boot-1", "result": "pass", "retry_counter": 0}] + dashboard_data = [ + {"id": "maestro:boot-1", "status": "PASS"}, + {"id": "maestro:dashboard-extra", "status": "NULL"}, + ] + + assert find_extra_dashboard_items(maestro_data, dashboard_data, "boot", False) == [ + "dashboard-extra" + ] + + def test_boot_stats_uses_retry_filtered_maestro_count_and_status(monkeypatch): helper_module = importlib.import_module( "kcidev.subcommands.maestro.validate.helper" @@ -465,3 +517,4 @@ def test_boot_stats_uses_retry_filtered_maestro_count_and_status(monkeypatch): assert stats[2] == 1 assert stats[3] == 2 assert stats[6] == [] + assert stats[7] == ["boot-retry"] diff --git a/tests/test_maestro_validate_stats.py b/tests/test_maestro_validate_stats.py index c09f950..69bc25d 100644 --- a/tests/test_maestro_validate_stats.py +++ b/tests/test_maestro_validate_stats.py @@ -31,4 +31,5 @@ def test_equal_counts_with_different_ids_fail_validation( stats = get_stats(None, "url", "branch", "commit", "tree", False, None) assert stats[2:5] == [1, 1, "❌"] - assert stats[5] == ["dashboard-only"] + assert stats[5] == ["maestro-only"] + assert stats[7] == ["dashboard-only"] From a26d110b574fbfc6f22ce589ad526094bbba9b68 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 15 Oct 2026 13:14:21 +0300 Subject: [PATCH 4/6] Only treat failed baseline boots as pending retries Boot validation dropped every failed Maestro boot with retry_counter < 3, assuming job-retry would replace it. job-retry only retries failed baseline jobs, and the KCIDB bridge sends failed non-baseline boot jobs to KCIDB on the first attempt. Validating such jobs would report them as extra in the dashboard. Drop failed boots pending retry only for baseline jobs, as the pipeline does. Signed-off-by: Denys Fedoryshchenko --- docs/maestro-validate.md | 2 +- kcidev/subcommands/maestro/validate/helper.py | 8 +++- tests/test_maestro_validate_json.py | 46 ++++++++++++++++++- 3 files changed, 51 insertions(+), 5 deletions(-) diff --git a/docs/maestro-validate.md b/docs/maestro-validate.md index aa86341..e208db1 100644 --- a/docs/maestro-validate.md +++ b/docs/maestro-validate.md @@ -89,7 +89,7 @@ Invoking `kci-dev maestro validate` without a subcommand prints help and exits w * **MISS** if error_code ∈ {`Bug`, `Configuration`, `invalid_job_params`, `Job`, `job_generation_error`, `ObjectNotPersisted`, `RequestBodyTooLarge`, `submit_error`, `Unexisting permission codename.`, `kbuild_internal_error`} * **ERROR** if error_code ∈ {`Canceled`, `Infrastructure`, `LAVATimeout`, `MultinodeTimeout`, `Test`} -* Boot retries excluded unless `retry_counter == 3`. +* Boot retries excluded unless `retry_counter == 3`: `incomplete` boots, and `fail` boots of `baseline*` jobs (the only failed jobs Maestro retries). * A **status mismatch** is reported when the mapped result disagrees with the dashboard. --- diff --git a/kcidev/subcommands/maestro/validate/helper.py b/kcidev/subcommands/maestro/validate/helper.py index cca75e4..94e2834 100644 --- a/kcidev/subcommands/maestro/validate/helper.py +++ b/kcidev/subcommands/maestro/validate/helper.py @@ -72,11 +72,15 @@ def filter_maestro_items_for_validation(maestro_data, item_type): if (b["result"] != "incomplete" or b["retry_counter"] == 3) ] if item_type == "boot": - # Exclude boot retries + # Exclude boot retries; job-retry only retries failed baseline jobs return [ b for b in maestro_data - if (b["result"] not in {"incomplete", "fail"} or b["retry_counter"] == 3) + if b["retry_counter"] == 3 + or not ( + b["result"] == "incomplete" + or (b["result"] == "fail" and b["name"].startswith("baseline")) + ) ] return maestro_data diff --git a/tests/test_maestro_validate_json.py b/tests/test_maestro_validate_json.py index fdcdc31..6875870 100644 --- a/tests/test_maestro_validate_json.py +++ b/tests/test_maestro_validate_json.py @@ -488,8 +488,20 @@ def test_boot_stats_uses_retry_filtered_maestro_count_and_status(monkeypatch): "kcidev.subcommands.maestro.validate.helper" ) maestro_boots = [ - {"id": "boot-1", "result": "pass", "retry_counter": 0, "data": {}}, - {"id": "boot-retry", "result": "fail", "retry_counter": 1, "data": {}}, + { + "id": "boot-1", + "name": "baseline-x86", + "result": "pass", + "retry_counter": 0, + "data": {}, + }, + { + "id": "boot-retry", + "name": "baseline-x86", + "result": "fail", + "retry_counter": 1, + "data": {}, + }, ] dashboard_boots = [ {"id": "maestro:boot-1", "status": "PASS"}, @@ -518,3 +530,33 @@ def test_boot_stats_uses_retry_filtered_maestro_count_and_status(monkeypatch): assert stats[3] == 2 assert stats[6] == [] assert stats[7] == ["boot-retry"] + + +def test_boot_retry_filter_keeps_failed_cip_jobs(): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + maestro_boots = [ + { + "id": "cip-fail", + "name": "job-gcc-14-x86-612-cip", + "result": "fail", + "retry_counter": 0, + }, + { + "id": "cip-retry", + "name": "job-gcc-14-x86-612-cip", + "result": "incomplete", + "retry_counter": 0, + }, + { + "id": "baseline-retry", + "name": "baseline-x86", + "result": "fail", + "retry_counter": 0, + }, + ] + + filtered = helper_module.filter_maestro_items_for_validation(maestro_boots, "boot") + + assert [b["id"] for b in filtered] == ["cip-fail"] From 498988ee73f728ca90d09ada15373fbb92407330 Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 15 Oct 2026 13:14:21 +0300 Subject: [PATCH 5/6] Fix: count CIP boot jobs in Maestro boot validation CIP boot jobs are named job-*-cip instead of baseline-*, but the pipeline sends them to KCIDB as boots. The Maestro boot query only matched ^baseline, so CIP checkouts reported 0 Maestro boots and every dashboard boot as extra. Match job-*-cip too. Signed-off-by: Denys Fedoryshchenko --- kcidev/subcommands/maestro/validate/helper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kcidev/subcommands/maestro/validate/helper.py b/kcidev/subcommands/maestro/validate/helper.py index 94e2834..1f79ed0 100644 --- a/kcidev/subcommands/maestro/validate/helper.py +++ b/kcidev/subcommands/maestro/validate/helper.py @@ -495,8 +495,10 @@ def get_boots(ctx, giturl, branch, commit, arch, raise_errors=False): """Get boots matching git URL, branch, and commit""" maestro_boots = [] dashboard_boots = [] + # CIP boot jobs are named job-*-cip rather than baseline-*, but are + # still sent to KCIDB as boots (kcidb_test_suite: boot / boot.nfs) filters = [ - "name__re=^baseline", + "name__re=^(baseline|job-.*-cip)", "data.kernel_revision.url=" + giturl, "data.kernel_revision.branch=" + branch, "data.kernel_revision.commit=" + commit, From 57b836fbd31964cc8ec874a131775cd653ec8e5c Mon Sep 17 00:00:00 2001 From: Denys Fedoryshchenko Date: Thu, 15 Oct 2026 13:14:21 +0300 Subject: [PATCH 6/6] Select Maestro boot jobs from the pipeline config Maestro nodes carry nothing that marks a job as a boot; the pipeline's KCIDB bridge decides that from each job's kcidb_test_suite. Matching boot jobs by name breaks silently whenever a boot job does not follow the expected naming. Add --pipeline-config to "maestro validate boots". It reads the kernelci-pipeline YAML config and queries Maestro for exactly the jobs whose kcidb_test_suite starts with "boot". Without the option, keep matching by name. Signed-off-by: Denys Fedoryshchenko --- docs/maestro-validate.md | 7 ++ kcidev/subcommands/maestro/validate/boots.py | 11 +++ kcidev/subcommands/maestro/validate/helper.py | 68 +++++++++++++++++-- tests/test_maestro_validate_json.py | 58 +++++++++++++++- 4 files changed, 136 insertions(+), 8 deletions(-) diff --git a/docs/maestro-validate.md b/docs/maestro-validate.md index e208db1..700dbbf 100644 --- a/docs/maestro-validate.md +++ b/docs/maestro-validate.md @@ -187,6 +187,9 @@ kci-dev maestro validate boots [OPTIONS] | `--arch` | str | | Architecture filter. | | `--verbose` | flag | `false` | Print detailed lists of missing items (JSON). | | `--table-output` | flag | `false` | Show a table (otherwise prints a simple list). | +| `--pipeline-config` | path | | kernelci-pipeline config directory or file. Boot jobs are the jobs whose `kcidb_test_suite` starts with `boot`. | + +Without `--pipeline-config`, Maestro boot jobs are matched by name: `baseline*` and `job-*-cip`. ### Output (Boots) @@ -245,6 +248,10 @@ kci-dev maestro validate boots \ --commit \ --arch x86_64 +# All boots for all trees, selecting boot jobs from the pipeline config +kci-dev maestro validate boots --all-checkouts \ + --pipeline-config /config + # Build history for a stable branch over the last 14 days kci-dev maestro validate builds --history \ --giturl https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git \ diff --git a/kcidev/subcommands/maestro/validate/boots.py b/kcidev/subcommands/maestro/validate/boots.py index b346f76..11afb8b 100644 --- a/kcidev/subcommands/maestro/validate/boots.py +++ b/kcidev/subcommands/maestro/validate/boots.py @@ -7,6 +7,7 @@ from .helper import ( get_boot_stats, + load_boot_job_names, print_simple_list, print_table_stats, print_validation_human, @@ -32,6 +33,7 @@ Examples: kci-dev maestro validate boots --all-checkouts --days kci-dev maestro validate boots -commit --giturl --branch + kci-dev maestro validate boots --all-checkouts --pipeline-config /config """, ) @click.option( @@ -97,6 +99,11 @@ default=False, help="Print a human-readable validation summary. With --json, text is written to stderr.", ) +@click.option( + "--pipeline-config", + type=click.Path(exists=True), + help="kernelci-pipeline config directory or file used to select boot jobs by kcidb_test_suite. Without it, boot jobs are matched by name.", +) @click.option( "--fail-on-mismatch", is_flag=True, @@ -119,9 +126,11 @@ def boots( table_output, use_json, human_readable, + pipeline_config, fail_on_mismatch, ): final_stats = [] + boot_jobs = load_boot_job_names(pipeline_config) if pipeline_config else None raise_errors = use_json or fail_on_mismatch stdout = sys.stdout if use_json: @@ -149,6 +158,7 @@ def boots( verbose, arch, raise_errors, + boot_jobs=boot_jobs, ) if stats: final_stats.append(stats) @@ -166,6 +176,7 @@ def boots( verbose, arch, raise_errors, + boot_jobs=boot_jobs, ) if stats: final_stats.append(stats) diff --git a/kcidev/subcommands/maestro/validate/helper.py b/kcidev/subcommands/maestro/validate/helper.py index 1f79ed0..beb0cb4 100644 --- a/kcidev/subcommands/maestro/validate/helper.py +++ b/kcidev/subcommands/maestro/validate/helper.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 import json +import os +import re from datetime import datetime, timedelta import click +import yaml from tabulate import tabulate from kcidev.libs.common import kci_msg, kci_msg_bold, kci_msg_json, kci_msg_red @@ -491,14 +494,59 @@ def validate_boot_status(maestro_boots, dashboard_boots): return boots_with_status_mismatch -def get_boots(ctx, giturl, branch, commit, arch, raise_errors=False): +def iter_yaml_files(config_path): + """Yield a single YAML file, or all YAML files under a directory""" + if os.path.isfile(config_path): + yield config_path + return + for root, dirs, files in os.walk(config_path): + dirs.sort() + for name in sorted(files): + if name.endswith((".yaml", ".yml")): + yield os.path.join(root, name) + + +def load_boot_job_names(config_path): + """ + Return names of jobs reported to KCIDB as boots. + + Read the kernelci-pipeline YAML config (a directory or a single file) and + select jobs whose kcidb_test_suite starts with "boot", the same rule the + pipeline's KCIDB bridge uses to classify boots. + """ + jobs = {} + for path in iter_yaml_files(config_path): + with open(path, encoding="utf-8") as f: + data = yaml.safe_load(f) + if isinstance(data, dict) and isinstance(data.get("jobs"), dict): + jobs.update(data["jobs"]) + boot_jobs = sorted( + name + for name, job in jobs.items() + if isinstance(job, dict) + and str(job.get("kcidb_test_suite") or "").startswith("boot") + ) + if not boot_jobs: + raise click.UsageError(f"No boot jobs found in pipeline config: {config_path}") + return boot_jobs + + +def boot_name_filter(boot_jobs=None): + """Return the Maestro node name filter matching boot jobs""" + if not boot_jobs: + # CIP boot jobs are named job-*-cip rather than baseline-*, but are + # still sent to KCIDB as boots (kcidb_test_suite: boot / boot.nfs) + return "name__re=^(baseline|job-.*-cip)" + names = "|".join(re.escape(name) for name in boot_jobs) + return f"name__re=^({names})$" + + +def get_boots(ctx, giturl, branch, commit, arch, raise_errors=False, boot_jobs=None): """Get boots matching git URL, branch, and commit""" maestro_boots = [] dashboard_boots = [] - # CIP boot jobs are named job-*-cip rather than baseline-*, but are - # still sent to KCIDB as boots (kcidb_test_suite: boot / boot.nfs) filters = [ - "name__re=^(baseline|job-.*-cip)", + boot_name_filter(boot_jobs), "data.kernel_revision.url=" + giturl, "data.kernel_revision.branch=" + branch, "data.kernel_revision.commit=" + commit, @@ -540,11 +588,19 @@ def get_boots(ctx, giturl, branch, commit, arch, raise_errors=False): def get_boot_stats( - ctx, giturl, branch, commit, tree_name, verbose, arch, raise_errors=False + ctx, + giturl, + branch, + commit, + tree_name, + verbose, + arch, + raise_errors=False, + boot_jobs=None, ): """Get boot stats""" maestro_boots, dashboard_boots = get_boots( - ctx, giturl, branch, commit, arch, raise_errors + ctx, giturl, branch, commit, arch, raise_errors, boot_jobs=boot_jobs ) if dashboard_boots is None: return [] diff --git a/tests/test_maestro_validate_json.py b/tests/test_maestro_validate_json.py index 6875870..0ae252c 100644 --- a/tests/test_maestro_validate_json.py +++ b/tests/test_maestro_validate_json.py @@ -2,6 +2,7 @@ import json import click +import pytest from click.testing import CliRunner from kcidev.subcommands.maestro.validate.helper import ( @@ -87,7 +88,7 @@ def _patch_boots_command(monkeypatch, stats): monkeypatch.setattr( boots_module, "get_boot_stats", - lambda ctx, giturl, branch, commit, tree_name, verbose, arch, raise_errors=False: stats, + lambda ctx, giturl, branch, commit, tree_name, verbose, arch, raise_errors=False, boot_jobs=None: stats, ) return boots_module.boots @@ -510,7 +511,7 @@ def test_boot_stats_uses_retry_filtered_maestro_count_and_status(monkeypatch): monkeypatch.setattr( helper_module, "get_boots", - lambda ctx, giturl, branch, commit, arch, raise_errors=False: ( + lambda ctx, giturl, branch, commit, arch, raise_errors=False, boot_jobs=None: ( maestro_boots, dashboard_boots, ), @@ -560,3 +561,56 @@ def test_boot_retry_filter_keeps_failed_cip_jobs(): filtered = helper_module.filter_maestro_items_for_validation(maestro_boots, "boot") assert [b["id"] for b in filtered] == ["cip-fail"] + + +def test_load_boot_job_names_uses_kcidb_test_suite(tmp_path): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + (tmp_path / "jobs.yaml").write_text( + """ +jobs: + baseline-x86: + kcidb_test_suite: boot + kbuild-gcc-14-x86: + kind: kbuild + kselftest-dt: + kcidb_test_suite: kselftest.dt +""" + ) + (tmp_path / "cip").mkdir() + (tmp_path / "cip" / "jobs-cip.yaml").write_text( + """ +_anchors: + boot-job: &boot-job + kind: job + kcidb_test_suite: boot.nfs +jobs: + job-gcc-14-x86-612-cip: *boot-job +""" + ) + + boot_jobs = helper_module.load_boot_job_names(str(tmp_path)) + + assert boot_jobs == ["baseline-x86", "job-gcc-14-x86-612-cip"] + assert helper_module.boot_name_filter(boot_jobs) == ( + r"name__re=^(baseline\-x86|job\-gcc\-14\-x86\-612\-cip)$" + ) + + +def test_load_boot_job_names_rejects_config_without_boot_jobs(tmp_path): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + (tmp_path / "jobs.yaml").write_text("jobs:\n kbuild-x86:\n kind: kbuild\n") + + with pytest.raises(click.UsageError): + helper_module.load_boot_job_names(str(tmp_path)) + + +def test_boot_name_filter_falls_back_to_name_regex(): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + + assert helper_module.boot_name_filter() == "name__re=^(baseline|job-.*-cip)"