diff --git a/docs/maestro-validate.md b/docs/maestro-validate.md index aa86341..700dbbf 100644 --- a/docs/maestro-validate.md +++ b/docs/maestro-validate.md @@ -89,7 +89,7 @@ Invoking `kci-dev maestro validate` without a subcommand prints help and exits w * **MISS** if error_code ∈ {`Bug`, `Configuration`, `invalid_job_params`, `Job`, `job_generation_error`, `ObjectNotPersisted`, `RequestBodyTooLarge`, `submit_error`, `Unexisting permission codename.`, `kbuild_internal_error`} * **ERROR** if error_code ∈ {`Canceled`, `Infrastructure`, `LAVATimeout`, `MultinodeTimeout`, `Test`} -* Boot retries excluded unless `retry_counter == 3`. +* Boot retries excluded unless `retry_counter == 3`: `incomplete` boots, and `fail` boots of `baseline*` jobs (the only failed jobs Maestro retries). * A **status mismatch** is reported when the mapped result disagrees with the dashboard. --- @@ -187,6 +187,9 @@ kci-dev maestro validate boots [OPTIONS] | `--arch` | str | | Architecture filter. | | `--verbose` | flag | `false` | Print detailed lists of missing items (JSON). | | `--table-output` | flag | `false` | Show a table (otherwise prints a simple list). | +| `--pipeline-config` | path | | kernelci-pipeline config directory or file. Boot jobs are the jobs whose `kcidb_test_suite` starts with `boot`. | + +Without `--pipeline-config`, Maestro boot jobs are matched by name: `baseline*` and `job-*-cip`. ### Output (Boots) @@ -245,6 +248,10 @@ kci-dev maestro validate boots \ --commit \ --arch x86_64 +# All boots for all trees, selecting boot jobs from the pipeline config +kci-dev maestro validate boots --all-checkouts \ + --pipeline-config /config + # Build history for a stable branch over the last 14 days kci-dev maestro validate builds --history \ --giturl https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git \ diff --git a/kcidev/subcommands/maestro/validate/boots.py b/kcidev/subcommands/maestro/validate/boots.py index 7c28586..11afb8b 100644 --- a/kcidev/subcommands/maestro/validate/boots.py +++ b/kcidev/subcommands/maestro/validate/boots.py @@ -7,8 +7,10 @@ from .helper import ( get_boot_stats, + load_boot_job_names, print_simple_list, print_table_stats, + print_validation_human, print_validation_json, validation_rows_to_json, ) @@ -31,6 +33,7 @@ Examples: kci-dev maestro validate boots --all-checkouts --days kci-dev maestro validate boots -commit --giturl --branch + kci-dev maestro validate boots --all-checkouts --pipeline-config /config """, ) @click.option( @@ -90,6 +93,17 @@ default=False, help="Print validation results as JSON", ) +@click.option( + "--human-readable", + is_flag=True, + default=False, + help="Print a human-readable validation summary. With --json, text is written to stderr.", +) +@click.option( + "--pipeline-config", + type=click.Path(exists=True), + help="kernelci-pipeline config directory or file used to select boot jobs by kcidb_test_suite. Without it, boot jobs are matched by name.", +) @click.option( "--fail-on-mismatch", is_flag=True, @@ -111,9 +125,12 @@ def boots( verbose, table_output, use_json, + human_readable, + pipeline_config, fail_on_mismatch, ): final_stats = [] + boot_jobs = load_boot_job_names(pipeline_config) if pipeline_config else None raise_errors = use_json or fail_on_mismatch stdout = sys.stdout if use_json: @@ -141,6 +158,7 @@ def boots( verbose, arch, raise_errors, + boot_jobs=boot_jobs, ) if stats: final_stats.append(stats) @@ -158,6 +176,7 @@ def boots( verbose, arch, raise_errors, + boot_jobs=boot_jobs, ) if stats: final_stats.append(stats) @@ -165,6 +184,8 @@ def boots( sys.stdout = stdout if use_json: report = print_validation_json("boots", final_stats, False, origin, days, arch) + if human_readable: + print_validation_human(report, err=True) if fail_on_mismatch and not report["summary"]["ok"]: ctx.exit(1) return @@ -177,11 +198,17 @@ def boots( "Boot count\ncomparison", "Missing boot IDs", "Boots with\nstatus mismatch", + "Extra dashboard\nboot IDs", ] - max_col_width = [None, 40, 3, 3, 2, 30, 30] + max_col_width = [None, 40, 3, 3, 2, 30, 30, 30] table_fmt = "simple_grid" if table_output: print_table_stats(final_stats, headers, max_col_width, table_fmt) + elif human_readable: + report = validation_rows_to_json( + "boots", final_stats, False, origin, days, arch + ) + print_validation_human(report) else: print_simple_list(final_stats, "boots", False) if fail_on_mismatch: diff --git a/kcidev/subcommands/maestro/validate/builds.py b/kcidev/subcommands/maestro/validate/builds.py index 248bb4c..beee270 100644 --- a/kcidev/subcommands/maestro/validate/builds.py +++ b/kcidev/subcommands/maestro/validate/builds.py @@ -10,6 +10,7 @@ get_builds_history_stats, print_simple_list, print_table_stats, + print_validation_human, print_validation_json, validation_rows_to_json, ) @@ -104,6 +105,12 @@ default=False, help="Print validation results as JSON", ) +@click.option( + "--human-readable", + is_flag=True, + default=False, + help="Print a human-readable validation summary. With --json, text is written to stderr.", +) @click.option( "--fail-on-mismatch", is_flag=True, @@ -126,6 +133,7 @@ def builds( verbose, table_output, use_json, + human_readable, fail_on_mismatch, ): final_stats = [] @@ -213,6 +221,8 @@ def builds( report = print_validation_json( "builds", final_stats, history, origin, days, arch ) + if human_readable: + print_validation_human(report, err=True) if fail_on_mismatch and not report["summary"]["ok"]: ctx.exit(1) return @@ -225,8 +235,9 @@ def builds( "Dashboard\nbuilds", "Build count\ncomparison", "Missing build IDs", + "Extra dashboard\nbuild IDs", ] - max_col_width = [None, 40, 3, 3, 2, 30] + max_col_width = [None, 40, 3, 3, 2, 30, 30] else: headers = [ @@ -237,11 +248,17 @@ def builds( "Build count\ncomparison", "Missing build IDs", "Builds with\nstatus mismatch", + "Extra dashboard\nbuild IDs", ] - max_col_width = [None, 40, 3, 3, 2, 30, 30] + max_col_width = [None, 40, 3, 3, 2, 30, 30, 30] table_fmt = "simple_grid" if table_output: print_table_stats(final_stats, headers, max_col_width, table_fmt) + elif human_readable: + report = validation_rows_to_json( + "builds", final_stats, history, origin, days, arch + ) + print_validation_human(report) else: print_simple_list(final_stats, "builds", history) if fail_on_mismatch: diff --git a/kcidev/subcommands/maestro/validate/helper.py b/kcidev/subcommands/maestro/validate/helper.py index 9cf5bb9..beb0cb4 100644 --- a/kcidev/subcommands/maestro/validate/helper.py +++ b/kcidev/subcommands/maestro/validate/helper.py @@ -1,9 +1,12 @@ #!/usr/bin/env python3 import json +import os +import re from datetime import datetime, timedelta import click +import yaml from tabulate import tabulate from kcidev.libs.common import kci_msg, kci_msg_bold, kci_msg_json, kci_msg_red @@ -57,37 +60,44 @@ def get_builds(ctx, giturl, branch, commit, arch, raise_errors=False): return maestro_builds, dashboard_builds -def find_missing_items(maestro_data, dashboard_data, item_type, verbose): +def filter_maestro_items_for_validation(maestro_data, item_type): """ - Compare build/boot IDs found in maestro with dashboard results - Return missing builds/boots in dashboard. - """ - dashboard_ids = [b["id"].split(":")[1] for b in dashboard_data] + Return Maestro items that should be compared against dashboard results. + Retry filtering must be shared by counts, missing-item checks, and status + checks so all validation fields describe the same item set. + """ if item_type == "build": # Exclude build retries - maestro_data = [ + return [ b for b in maestro_data if (b["result"] != "incomplete" or b["retry_counter"] == 3) ] - elif item_type == "boot": - # Exclude boot retries - maestro_data = [ + if item_type == "boot": + # Exclude boot retries; job-retry only retries failed baseline jobs + return [ b for b in maestro_data - if (b["result"] not in {"incomplete", "fail"} or b["retry_counter"] == 3) + if b["retry_counter"] == 3 + or not ( + b["result"] == "incomplete" + or (b["result"] == "fail" and b["name"].startswith("baseline")) + ) ] + return maestro_data - maestro_ids = [b["id"] for b in maestro_data] - if len(maestro_ids) > len(dashboard_ids): - missing_items = [b for b in maestro_data if b["id"] not in dashboard_ids] - missing_ids = [b["id"] for b in missing_items] - else: - missing_items = [ - b for b in dashboard_data if b["id"].split(":")[1] not in maestro_ids - ] - missing_ids = [b["id"].split(":")[1] for b in missing_items] + +def find_missing_items(maestro_data, dashboard_data, item_type, verbose): + """ + Compare build/boot IDs found in maestro with dashboard results + Return missing builds/boots in dashboard. + """ + maestro_data = filter_maestro_items_for_validation(maestro_data, item_type) + dashboard_ids = [b["id"].split(":")[1] for b in dashboard_data] + + missing_items = [b for b in maestro_data if b["id"] not in dashboard_ids] + missing_ids = [b["id"] for b in missing_items] if missing_items and verbose: kci_msg("Missing items:") @@ -96,6 +106,26 @@ def find_missing_items(maestro_data, dashboard_data, item_type, verbose): return missing_ids +def find_extra_dashboard_items(maestro_data, dashboard_data, item_type, verbose): + """ + Compare build/boot IDs found in dashboard with Maestro results. + Return dashboard builds/boots that are not present in Maestro. + """ + maestro_data = filter_maestro_items_for_validation(maestro_data, item_type) + maestro_ids = {b["id"] for b in maestro_data} + + extra_items = [ + b for b in dashboard_data if b["id"].split(":")[1] not in maestro_ids + ] + extra_ids = [b["id"].split(":")[1] for b in extra_items] + + if extra_items and verbose: + kci_msg("Extra dashboard items:") + kci_msg_json(extra_items) + + return extra_ids + + def validate_build_status(maestro_builds, dashboard_builds): """ Validate if the build status of dashboard pulled build @@ -127,17 +157,21 @@ def get_build_stats( ) if dashboard_builds is None: return [] + maestro_builds = filter_maestro_items_for_validation(maestro_builds, "build") summary_flag = "✅" missing_build_ids = find_missing_items( maestro_builds, dashboard_builds, "build", verbose ) - + extra_dashboard_build_ids = find_extra_dashboard_items( + maestro_builds, dashboard_builds, "build", verbose + ) builds_with_status_mismatch = validate_build_status( maestro_builds, dashboard_builds ) if ( len(dashboard_builds) != len(maestro_builds) or missing_build_ids + or extra_dashboard_build_ids or builds_with_status_mismatch ): summary_flag = "❌" @@ -149,6 +183,7 @@ def get_build_stats( summary_flag, missing_build_ids, builds_with_status_mismatch, + extra_dashboard_build_ids, ] return stats @@ -169,7 +204,15 @@ def extract_validation_data(row: list): comparison = row[4] # Build/boot count comparison (✅ or ❌) missing_ids = row[5] # Missing build/boot IDs mismatched_ids = row[6] - return tree_branch, commit, comparison, missing_ids, mismatched_ids + extra_dashboard_ids = row[7] or [] if len(row) > 7 else [] + return ( + tree_branch, + commit, + comparison, + missing_ids, + mismatched_ids, + extra_dashboard_ids, + ) except IndexError: kci_msg_red("Failed to extract data for list view report") raise ValueError() @@ -192,6 +235,7 @@ def print_simple_list(data, item_type, history=False): comparison, missing_ids, mismatched_ids, + extra_dashboard_ids, ) = extract_validation_data(row) except ValueError: continue @@ -201,6 +245,7 @@ def print_simple_list(data, item_type, history=False): "status": comparison, "missing_ids": missing_ids, "mismatched_ids": mismatched_ids, + "extra_dashboard_ids": extra_dashboard_ids, } ) @@ -219,6 +264,12 @@ def print_simple_list(data, item_type, history=False): ) for id in commit["mismatched_ids"]: kci_msg(f" - https://api.kernelci.org/viewer?node_id={id}") + if commit["extra_dashboard_ids"]: + kci_msg( + f" Extra dashboard {item_type}: {len(commit['extra_dashboard_ids'])}" + ) + for id in commit["extra_dashboard_ids"]: + kci_msg(f" - https://api.kernelci.org/viewer?node_id={id}") elif commit["status"] == "✅": kci_msg(f" Commit {commit['commit'][:12]}: ✅") @@ -232,6 +283,7 @@ def print_simple_list(data, item_type, history=False): comparison, missing_ids, mismatched_ids, + extra_dashboard_ids, ) = extract_validation_data(row) except ValueError: continue @@ -247,6 +299,10 @@ def print_simple_list(data, item_type, history=False): kci_msg(f" Status mismatched {item_type}: {len(mismatched_ids)}") for id in mismatched_ids: kci_msg(f" - https://api.kernelci.org/viewer?node_id={id}") + if comparison == "❌" and extra_dashboard_ids: + kci_msg(f" Extra dashboard {item_type}: {len(extra_dashboard_ids)}") + for id in extra_dashboard_ids: + kci_msg(f" - https://api.kernelci.org/viewer?node_id={id}") kci_msg("") @@ -256,8 +312,9 @@ def validation_rows_to_json(kind, rows, history, origin, days, arch): for row in rows: missing_ids = row[5] or [] status_mismatch_ids = row[6] or [] + extra_dashboard_ids = row[7] or [] if len(row) > 7 else [] ok = row[2] == row[3] and row[4] == "✅" and not missing_ids - ok = ok and not status_mismatch_ids + ok = ok and not status_mismatch_ids and not extra_dashboard_ids results.append( { "tree_branch": row[0], @@ -266,6 +323,7 @@ def validation_rows_to_json(kind, rows, history, origin, days, arch): "dashboard_count": row[3], "ok": ok, "missing_ids": missing_ids, + "extra_in_dashboard_ids": extra_dashboard_ids, "status_mismatch_ids": status_mismatch_ids, } ) @@ -280,6 +338,9 @@ def validation_rows_to_json(kind, rows, history, origin, days, arch): "checked": len(results), "failed": sum(not result["ok"] for result in results), "missing": sum(len(result["missing_ids"]) for result in results), + "extra_in_dashboard": sum( + len(result["extra_in_dashboard_ids"]) for result in results + ), "status_mismatches": sum( len(result["status_mismatch_ids"]) for result in results ), @@ -289,6 +350,92 @@ def validation_rows_to_json(kind, rows, history, origin, days, arch): } +def validation_report_to_text(report): + """Render a JSON validation report as human-readable text.""" + item_type = report["kind"][:-1] if report["kind"].endswith("s") else report["kind"] + summary = report["summary"] + scope = [ + f"origin={report['origin']}", + f"days={report['days']}", + f"history={'yes' if report['history'] else 'no'}", + ] + if report.get("arch"): + scope.append(f"arch={report['arch']}") + else: + scope.append("arch=all") + + status = "PASS" if summary["ok"] else "FAIL" + lines = [ + f"Dashboard validation: {report['kind']} ({status})", + f"Scope: {', '.join(scope)}", + ( + "Summary: " + f"checked={summary['checked']}, " + f"failed={summary['failed']}, " + f"missing={summary['missing']}, " + f"extra_in_dashboard={summary['extra_in_dashboard']}, " + f"status_mismatches={summary['status_mismatches']}" + ), + ] + + if report["results"]: + lines.append("") + lines.append( + "Note: count comparisons use raw query totals; missing IDs are " + "reported after validation filtering. Extra dashboard IDs are " + "dashboard rows not present in the filtered Maestro set." + ) + + for result in report["results"]: + result_status = "PASS" if result["ok"] else "FAIL" + commit = result["commit"] + delta = result["maestro_count"] - result["dashboard_count"] + if delta > 0: + delta_text = f"{delta} more in Maestro" + elif delta < 0: + delta_text = f"{-delta} more in dashboard" + else: + delta_text = "counts match" + + lines.extend( + [ + "", + f"{result['tree_branch']} @ {commit[:12]}: {result_status}", + f" Commit: {commit}", + f" Maestro {report['kind']}: {result['maestro_count']}", + f" Dashboard {report['kind']}: {result['dashboard_count']}", + f" Count delta: {delta_text}", + ] + ) + + missing_ids = result["missing_ids"] + extra_dashboard_ids = result["extra_in_dashboard_ids"] + status_mismatch_ids = result["status_mismatch_ids"] + if missing_ids: + lines.append(f" Missing {item_type} IDs: {len(missing_ids)}") + for node_id in missing_ids: + lines.append(f" - https://api.kernelci.org/viewer?node_id={node_id}") + if extra_dashboard_ids: + lines.append( + f" Extra dashboard {item_type} IDs: {len(extra_dashboard_ids)}" + ) + for node_id in extra_dashboard_ids: + lines.append(f" - https://api.kernelci.org/viewer?node_id={node_id}") + if status_mismatch_ids: + lines.append( + f" Status mismatch {item_type} IDs: {len(status_mismatch_ids)}" + ) + for node_id in status_mismatch_ids: + lines.append(f" - https://api.kernelci.org/viewer?node_id={node_id}") + + return "\n".join(lines) + + +def print_validation_human(report, err=False): + """Print a human-readable validation report.""" + click.echo(validation_report_to_text(report), err=err) + + def print_validation_json(kind, rows, history, origin, days, arch): """Print a single JSON validation report to stdout.""" report = validation_rows_to_json(kind, rows, history, origin, days, arch) @@ -347,12 +494,59 @@ def validate_boot_status(maestro_boots, dashboard_boots): return boots_with_status_mismatch -def get_boots(ctx, giturl, branch, commit, arch, raise_errors=False): +def iter_yaml_files(config_path): + """Yield a single YAML file, or all YAML files under a directory""" + if os.path.isfile(config_path): + yield config_path + return + for root, dirs, files in os.walk(config_path): + dirs.sort() + for name in sorted(files): + if name.endswith((".yaml", ".yml")): + yield os.path.join(root, name) + + +def load_boot_job_names(config_path): + """ + Return names of jobs reported to KCIDB as boots. + + Read the kernelci-pipeline YAML config (a directory or a single file) and + select jobs whose kcidb_test_suite starts with "boot", the same rule the + pipeline's KCIDB bridge uses to classify boots. + """ + jobs = {} + for path in iter_yaml_files(config_path): + with open(path, encoding="utf-8") as f: + data = yaml.safe_load(f) + if isinstance(data, dict) and isinstance(data.get("jobs"), dict): + jobs.update(data["jobs"]) + boot_jobs = sorted( + name + for name, job in jobs.items() + if isinstance(job, dict) + and str(job.get("kcidb_test_suite") or "").startswith("boot") + ) + if not boot_jobs: + raise click.UsageError(f"No boot jobs found in pipeline config: {config_path}") + return boot_jobs + + +def boot_name_filter(boot_jobs=None): + """Return the Maestro node name filter matching boot jobs""" + if not boot_jobs: + # CIP boot jobs are named job-*-cip rather than baseline-*, but are + # still sent to KCIDB as boots (kcidb_test_suite: boot / boot.nfs) + return "name__re=^(baseline|job-.*-cip)" + names = "|".join(re.escape(name) for name in boot_jobs) + return f"name__re=^({names})$" + + +def get_boots(ctx, giturl, branch, commit, arch, raise_errors=False, boot_jobs=None): """Get boots matching git URL, branch, and commit""" maestro_boots = [] dashboard_boots = [] filters = [ - "name__re=^baseline", + boot_name_filter(boot_jobs), "data.kernel_revision.url=" + giturl, "data.kernel_revision.branch=" + branch, "data.kernel_revision.commit=" + commit, @@ -394,22 +588,35 @@ def get_boots(ctx, giturl, branch, commit, arch, raise_errors=False): def get_boot_stats( - ctx, giturl, branch, commit, tree_name, verbose, arch, raise_errors=False + ctx, + giturl, + branch, + commit, + tree_name, + verbose, + arch, + raise_errors=False, + boot_jobs=None, ): """Get boot stats""" maestro_boots, dashboard_boots = get_boots( - ctx, giturl, branch, commit, arch, raise_errors + ctx, giturl, branch, commit, arch, raise_errors, boot_jobs=boot_jobs ) if dashboard_boots is None: return [] + maestro_boots = filter_maestro_items_for_validation(maestro_boots, "boot") summary_flag = "✅" missing_boot_ids = find_missing_items( maestro_boots, dashboard_boots, "boot", verbose ) + extra_dashboard_boot_ids = find_extra_dashboard_items( + maestro_boots, dashboard_boots, "boot", verbose + ) boots_with_status_mismatch = validate_boot_status(maestro_boots, dashboard_boots) if ( len(dashboard_boots) != len(maestro_boots) or missing_boot_ids + or extra_dashboard_boot_ids or boots_with_status_mismatch ): summary_flag = "❌" @@ -421,6 +628,7 @@ def get_boot_stats( summary_flag, missing_boot_ids, boots_with_status_mismatch, + extra_dashboard_boot_ids, ] return stats @@ -432,14 +640,19 @@ def get_builds_history_stats( final_stats = [] builds_history = get_builds_history(ctx, checkouts, arch, raise_errors) for b in builds_history: - missing_build_ids = find_missing_items(b[1], b[2], "build", verbose) - total_maestro_builds = len(b[1]) + maestro_builds = filter_maestro_items_for_validation(b[1], "build") + missing_build_ids = find_missing_items(maestro_builds, b[2], "build", verbose) + extra_dashboard_build_ids = find_extra_dashboard_items( + maestro_builds, b[2], "build", verbose + ) + total_maestro_builds = len(maestro_builds) total_dashboard_builds = len(b[2]) - mismatched_ids = validate_build_status(b[1], b[2]) + mismatched_ids = validate_build_status(maestro_builds, b[2]) summary_flag = ( "✅" if total_maestro_builds == total_dashboard_builds and not missing_build_ids + and not extra_dashboard_build_ids and not mismatched_ids else "❌" ) @@ -451,6 +664,7 @@ def get_builds_history_stats( summary_flag, missing_build_ids, mismatched_ids, + extra_dashboard_build_ids, ] final_stats.append(stats) return final_stats diff --git a/tests/test_maestro_validate_json.py b/tests/test_maestro_validate_json.py index e70e4cc..0ae252c 100644 --- a/tests/test_maestro_validate_json.py +++ b/tests/test_maestro_validate_json.py @@ -2,9 +2,15 @@ import json import click +import pytest from click.testing import CliRunner -from kcidev.subcommands.maestro.validate.helper import validation_rows_to_json +from kcidev.subcommands.maestro.validate.helper import ( + find_extra_dashboard_items, + find_missing_items, + validation_report_to_text, + validation_rows_to_json, +) COMMIT = "a" * 40 @@ -13,12 +19,17 @@ def _stdout(result): return getattr(result, "stdout", result.output) +def _stderr(result): + return getattr(result, "stderr", "") + + def _build_row( maestro_count=1, dashboard_count=1, summary_flag="✅", missing_ids=None, status_mismatch_ids=None, + extra_dashboard_ids=None, ): return [ "mainline/master", @@ -28,6 +39,7 @@ def _build_row( summary_flag, missing_ids or [], status_mismatch_ids or [], + extra_dashboard_ids or [], ] @@ -76,7 +88,7 @@ def _patch_boots_command(monkeypatch, stats): monkeypatch.setattr( boots_module, "get_boot_stats", - lambda ctx, giturl, branch, commit, tree_name, verbose, arch, raise_errors=False: stats, + lambda ctx, giturl, branch, commit, tree_name, verbose, arch, raise_errors=False, boot_jobs=None: stats, ) return boots_module.boots @@ -176,6 +188,23 @@ def test_missing_ids_make_summary_not_ok(): assert report["results"][0]["ok"] is False +def test_extra_dashboard_ids_make_summary_not_ok(): + report = validation_rows_to_json( + "boots", + [_build_row(1, 2, "❌", extra_dashboard_ids=["boot-extra"])], + False, + "maestro", + 1, + None, + ) + + assert report["summary"]["ok"] is False + assert report["summary"]["failed"] == 1 + assert report["summary"]["extra_in_dashboard"] == 1 + assert report["results"][0]["extra_in_dashboard_ids"] == ["boot-extra"] + assert report["results"][0]["ok"] is False + + def test_status_mismatches_make_summary_not_ok(): report = validation_rows_to_json( "boots", @@ -218,6 +247,86 @@ def test_json_mismatches_still_exit_zero(monkeypatch): assert json.loads(_stdout(result))["summary"]["ok"] is False +def test_validation_report_to_text_decodes_summary(): + report = validation_rows_to_json( + "boots", + [ + _build_row( + 116, + 45, + "❌", + missing_ids=["boot-1"], + status_mismatch_ids=["boot-2"], + ) + ], + False, + "maestro", + 1, + None, + ) + + text = validation_report_to_text(report) + + assert "Dashboard validation: boots (FAIL)" in text + assert ( + "Summary: checked=1, failed=1, missing=1, extra_in_dashboard=0, " + "status_mismatches=1" + ) in text + assert "mainline/master @ aaaaaaaaaaaa: FAIL" in text + assert "Count delta: 71 more in Maestro" in text + assert "https://api.kernelci.org/viewer?node_id=boot-1" in text + assert "https://api.kernelci.org/viewer?node_id=boot-2" in text + + +def test_validation_report_to_text_shows_extra_dashboard_ids(): + report = validation_rows_to_json( + "boots", + [_build_row(1, 2, "❌", extra_dashboard_ids=["boot-extra"])], + False, + "maestro", + 1, + None, + ) + + text = validation_report_to_text(report) + + assert "Count delta: 1 more in dashboard" in text + assert "Extra dashboard boot IDs: 1" in text + assert "https://api.kernelci.org/viewer?node_id=boot-extra" in text + + +def test_json_human_readable_keeps_stdout_json_and_writes_text_to_stderr( + monkeypatch, +): + command = _patch_boots_command( + monkeypatch, + _build_row(2, 1, "❌", missing_ids=["boot-1"]), + ) + args = _json_args() + ["--human-readable"] + + result = CliRunner().invoke(command, args) + + assert result.exit_code == 0 + report = json.loads(_stdout(result)) + assert report["summary"]["ok"] is False + assert "Dashboard validation: boots (FAIL)" in _stderr(result) + assert "node_id=boot-1" in _stderr(result) + + +def test_human_readable_without_json_uses_decoded_text(monkeypatch): + command = _patch_builds_command( + monkeypatch, + _build_row(2, 1, "❌", missing_ids=["build-1"]), + ) + args = _json_args()[:-1] + ["--human-readable"] + + result = CliRunner().invoke(command, args) + + assert result.exit_code == 0 + assert "Dashboard validation: builds (FAIL)" in _stdout(result) + assert "node_id=build-1" in _stdout(result) + + def test_json_runtime_failures_exit_nonzero(monkeypatch): builds_module = importlib.import_module( "kcidev.subcommands.maestro.validate.builds" @@ -351,3 +460,157 @@ def test_mismatch_without_fail_on_mismatch_still_exits_zero(monkeypatch): result = CliRunner().invoke(command, args) assert result.exit_code == 0 + + +def test_missing_items_only_reports_maestro_items_missing_from_dashboard(): + maestro_data = [{"id": "boot-1", "result": "pass", "retry_counter": 0}] + dashboard_data = [ + {"id": "maestro:boot-1", "status": "PASS"}, + {"id": "maestro:dashboard-extra", "status": "PASS"}, + ] + + assert find_missing_items(maestro_data, dashboard_data, "boot", False) == [] + + +def test_extra_dashboard_items_report_dashboard_items_missing_from_maestro(): + maestro_data = [{"id": "boot-1", "result": "pass", "retry_counter": 0}] + dashboard_data = [ + {"id": "maestro:boot-1", "status": "PASS"}, + {"id": "maestro:dashboard-extra", "status": "NULL"}, + ] + + assert find_extra_dashboard_items(maestro_data, dashboard_data, "boot", False) == [ + "dashboard-extra" + ] + + +def test_boot_stats_uses_retry_filtered_maestro_count_and_status(monkeypatch): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + maestro_boots = [ + { + "id": "boot-1", + "name": "baseline-x86", + "result": "pass", + "retry_counter": 0, + "data": {}, + }, + { + "id": "boot-retry", + "name": "baseline-x86", + "result": "fail", + "retry_counter": 1, + "data": {}, + }, + ] + dashboard_boots = [ + {"id": "maestro:boot-1", "status": "PASS"}, + {"id": "maestro:boot-retry", "status": "PASS"}, + ] + monkeypatch.setattr( + helper_module, + "get_boots", + lambda ctx, giturl, branch, commit, arch, raise_errors=False, boot_jobs=None: ( + maestro_boots, + dashboard_boots, + ), + ) + + stats = helper_module.get_boot_stats( + None, + "https://example.com/linux.git", + "master", + COMMIT, + "mainline", + False, + None, + ) + + assert stats[2] == 1 + assert stats[3] == 2 + assert stats[6] == [] + assert stats[7] == ["boot-retry"] + + +def test_boot_retry_filter_keeps_failed_cip_jobs(): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + maestro_boots = [ + { + "id": "cip-fail", + "name": "job-gcc-14-x86-612-cip", + "result": "fail", + "retry_counter": 0, + }, + { + "id": "cip-retry", + "name": "job-gcc-14-x86-612-cip", + "result": "incomplete", + "retry_counter": 0, + }, + { + "id": "baseline-retry", + "name": "baseline-x86", + "result": "fail", + "retry_counter": 0, + }, + ] + + filtered = helper_module.filter_maestro_items_for_validation(maestro_boots, "boot") + + assert [b["id"] for b in filtered] == ["cip-fail"] + + +def test_load_boot_job_names_uses_kcidb_test_suite(tmp_path): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + (tmp_path / "jobs.yaml").write_text( + """ +jobs: + baseline-x86: + kcidb_test_suite: boot + kbuild-gcc-14-x86: + kind: kbuild + kselftest-dt: + kcidb_test_suite: kselftest.dt +""" + ) + (tmp_path / "cip").mkdir() + (tmp_path / "cip" / "jobs-cip.yaml").write_text( + """ +_anchors: + boot-job: &boot-job + kind: job + kcidb_test_suite: boot.nfs +jobs: + job-gcc-14-x86-612-cip: *boot-job +""" + ) + + boot_jobs = helper_module.load_boot_job_names(str(tmp_path)) + + assert boot_jobs == ["baseline-x86", "job-gcc-14-x86-612-cip"] + assert helper_module.boot_name_filter(boot_jobs) == ( + r"name__re=^(baseline\-x86|job\-gcc\-14\-x86\-612\-cip)$" + ) + + +def test_load_boot_job_names_rejects_config_without_boot_jobs(tmp_path): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + (tmp_path / "jobs.yaml").write_text("jobs:\n kbuild-x86:\n kind: kbuild\n") + + with pytest.raises(click.UsageError): + helper_module.load_boot_job_names(str(tmp_path)) + + +def test_boot_name_filter_falls_back_to_name_regex(): + helper_module = importlib.import_module( + "kcidev.subcommands.maestro.validate.helper" + ) + + assert helper_module.boot_name_filter() == "name__re=^(baseline|job-.*-cip)" diff --git a/tests/test_maestro_validate_stats.py b/tests/test_maestro_validate_stats.py index c09f950..69bc25d 100644 --- a/tests/test_maestro_validate_stats.py +++ b/tests/test_maestro_validate_stats.py @@ -31,4 +31,5 @@ def test_equal_counts_with_different_ids_fail_validation( stats = get_stats(None, "url", "branch", "commit", "tree", False, None) assert stats[2:5] == [1, 1, "❌"] - assert stats[5] == ["dashboard-only"] + assert stats[5] == ["maestro-only"] + assert stats[7] == ["dashboard-only"]