diff --git a/docs/contrib/Metrics.md b/docs/contrib/Metrics.md index 8b66f54670..004b2cb43d 100644 --- a/docs/contrib/Metrics.md +++ b/docs/contrib/Metrics.md @@ -57,16 +57,21 @@ artifact, is compared against that commit's own build. The dashboard finds the merge base through GitHub, so it cannot find one for a commit that was never pushed. The check handles that commit itself: when no remote branch in your checkout contains the commit, it computes -`git merge-base origin/master` locally and pins the baseline to that -build. A commit that is on a remote, or that is not in your checkout at all, is -left to the dashboard. - -When the dashboard has no `master` build at the merge base, or no merge base, -it compares against the latest `master` build instead (the latest one before -the commit, if it knows the commit's date). That build may be newer than the -commit's base, so changes merged since then can show up as regressions or hide -real ones, and the result moves with `master`. The report warns whenever this -happens; set `DASHBOARD_BASE_COMMIT` to compare against an exact build instead. +`git merge-base upstream/master` locally (`origin/master` when there +is no `upstream` remote) and pins the baseline to that build. The report warns +with the merge base and the ref it came from, because an outdated ref moves +the baseline. A commit that is on a remote, or that is not in your checkout at +all, is left to the dashboard. So is a commit with no merge base with the ref, +with a warning. + +When the pinned merge base has no build of the design, the check is sent again +without the pin. When the dashboard has no `master` build at the merge base, or +no merge base, it compares against the latest `master` build instead (the +latest one before the commit, if it knows the commit's date). That build may +be newer than the commit's base, so changes merged since then can show up as +regressions or hide real ones, and the result moves with `master`. The report +warns whenever this happens; set `DASHBOARD_BASE_COMMIT` to compare against an +exact build instead. These environment variables shape the comparison: @@ -85,9 +90,10 @@ These environment variables shape the comparison: it unset for a local run, private or not. With it set, an unpushed commit is not pinned to its local merge base. - `DASHBOARD_MERGE_BASE` (`--merge-base`) names the ref an unpushed commit's - merge base is computed against, `origin/master` by default. Set it to your - upstream ref if `origin` is a fork with an outdated `master`, or to `none` to - leave unpushed commits to the dashboard. It does not apply together with + merge base is computed against. The default is `upstream/master` when that + ref exists, else `origin/master`. Set it if your project remote has another + name, or to `none` to leave unpushed commits to the dashboard. It does not + apply together with `DASHBOARD_BASE_COMMIT` or `DASHBOARD_JOB_NAME`. - `DASHBOARD_BASE_COMMIT` (`--base-commit`) pins the baseline to the build of that commit. Nothing else is tried: if that commit has no build of the diff --git a/flow/test/test_checkQorMetrics.py b/flow/test/test_checkQorMetrics.py index 4363c5a8f2..5f31e50932 100644 --- a/flow/test/test_checkQorMetrics.py +++ b/flow/test/test_checkQorMetrics.py @@ -276,13 +276,70 @@ def test_commit_missing_from_the_checkout_is_left_to_the_dashboard(self): _, _, payloads = self.check(COMMIT) self.assertNotIn("BaseCommitSha", payloads[0]) - def test_unknown_ref_is_an_error(self): + def test_pin_warns_with_the_ref(self): + repository, master, unpushed = self.repository() + self.enter(repository) + _, output, _ = self.check(unpushed) + self.assertIn( + f"[WARN] {unpushed[:12]} is on no remote, so the baseline was pinned " + f"to its local merge base {master[:12]} with origin/master", + output, + ) + + def test_upstream_master_is_the_default_when_it_exists(self): + # A fork clone: origin/master is the fork's old master, upstream/master + # the project's, and the local commit is on the project's. + repository, old_master, _ = self.repository() + git(repository, "reset", "-q", "--hard", old_master) + git(repository, "commit", "-q", "--allow-empty", "-m", "upstream") + upstream = git(repository, "rev-parse", "HEAD") + git(repository, "update-ref", "refs/remotes/upstream/master", upstream) + git(repository, "commit", "-q", "--allow-empty", "-m", "local") + unpushed = git(repository, "rev-parse", "HEAD") + self.enter(repository) + _, output, payloads = self.check(unpushed) + self.assertEqual(payloads[0]["BaseCommitSha"], upstream) + self.assertIn(f"{upstream[:12]} with upstream/master", output) + + def test_no_merge_base_warns_and_sends_no_pin(self): repository, _, unpushed = self.repository() + git(repository, "checkout", "-q", "--orphan", "unrelated") + git(repository, "commit", "-q", "--allow-empty", "-m", "unrelated") + unrelated = git(repository, "rev-parse", "HEAD") + git(repository, "update-ref", "refs/remotes/upstream/master", unrelated) self.enter(repository) - code, output, payloads = self.check(unpushed, "--merge-base", "upstream/master") - self.assertEqual(code, 1) - self.assertEqual(payloads, []) - self.assertIn("has no merge base with upstream/master", output) + code, output, payloads = self.check(unpushed) + self.assertEqual(code, 0) + self.assertEqual(len(payloads), 1) + self.assertNotIn("BaseCommitSha", payloads[0]) + self.assertIn( + "[WARN] " + unpushed[:12] + " is on no remote and has no merge base " + "with upstream/master, so the baseline was not pinned", + output, + ) + self.assertIn("QoR check: PASS", output) + + def test_missing_ref_warns_and_sends_no_pin(self): + repository, _, unpushed = self.repository() + self.enter(repository) + code, output, payloads = self.check(unpushed, "--merge-base", "upstrem/master") + self.assertEqual(code, 0) + self.assertNotIn("BaseCommitSha", payloads[0]) + self.assertIn("[WARN] upstrem/master does not exist here", output) + + def test_failed_remote_lookup_sends_no_pin(self): + repository, _, unpushed = self.repository() + self.enter(repository) + real_git = checkQorMetrics.git + + def git_without_for_each_ref(*args): + return None if args[0] == "for-each-ref" else real_git(*args) + + with patch.object(checkQorMetrics, "git", git_without_for_each_ref): + code, output, payloads = self.check(unpushed) + self.assertEqual(code, 0) + self.assertNotIn("BaseCommitSha", payloads[0]) + self.assertIn(f"git could not tell whether {unpushed[:12]}", output) def test_ref_from_the_environment(self): repository, _, unpushed = self.repository() @@ -310,15 +367,40 @@ def test_job_name_skips_it(self): _, _, payloads = self.check(unpushed, "--job-name", "orfs-ci") self.assertNotIn("BaseCommitSha", payloads[0]) - def test_pinned_merge_base_without_a_build_suggests_turning_it_off(self): + def test_merge_base_without_a_build_is_sent_again_unpinned(self): + repository, master, unpushed = self.repository() + self.enter(repository) + code, output, payloads = self.run_check( + metadata=self.metadata(commit=unpushed), + dashboard=Dashboard( + response(status="no_baseline", strategy="none"), + response(strategy="latest-master"), + ), + ) + self.assertEqual(code, 0) + self.assertEqual(len(payloads), 2) + self.assertEqual(payloads[0]["BaseCommitSha"], master) + self.assertNotIn("BaseCommitSha", payloads[1]) + self.assertIn( + f"merge base {master[:12]} with origin/master has no build of this " + "design, so the check was sent again without that pin", + output, + ) + self.assertIn("compared against the latest master build", output) + self.assertIn("QoR check: PASS", output) + + def test_explicit_base_commit_without_a_build_is_not_sent_again(self): repository, _, unpushed = self.repository() self.enter(repository) - code, output, _ = self.run_check( + code, output, payloads = self.run_check( + "--base-commit", + BASE, metadata=self.metadata(commit=unpushed), dashboard=Dashboard(response(status="no_baseline", strategy="none")), ) self.assertEqual(code, 0) - self.assertIn("DASHBOARD_MERGE_BASE=none", output) + self.assertEqual(len(payloads), 1) + self.assertIn("QoR check: INCONCLUSIVE", output) class MetricFilterTest(CheckQorMetricsTest): @@ -411,17 +493,22 @@ def test_fail_lists_the_failed_rule(self): failed = { "metricName": "finish__timing__setup__ws", "status": "fail", - "baselineValue": 2, - "currentValue": 1.5, - "limit": 1.9, + "baseValue": 2, + "targetValue": 1.5, + "thresholdValue": 1.9, "deltaPct": -25, } code, output, _ = self.run_check( dashboard=Dashboard(response(status="fail", failed=1, metrics=[failed])) ) self.assertEqual(code, 1) - self.assertIn("finish__timing__setup__ws", output) - self.assertIn("-25.00", output) + self.assertIn( + checkQorMetrics.ROW_FMT.format( + "finish__timing__setup__ws", "2", "1.5000", "1.9000", "-25.00" + ), + output, + ) + self.assertIn("finish__timing__setup__ws 1.5 (base 2, limit 1.9)", output) self.assertIn("QoR check: FAIL", output) def test_latest_master_warns_with_a_commit(self): diff --git a/flow/util/checkQorMetrics.py b/flow/util/checkQorMetrics.py index 8a0f4a85f4..9e01f3d7c1 100755 --- a/flow/util/checkQorMetrics.py +++ b/flow/util/checkQorMetrics.py @@ -27,10 +27,11 @@ master. A commit on master is its own merge base, so a master run is compared against that commit's own build. The dashboard finds the merge base through GitHub, so for a commit that no remote branch here contains, the script -computes it locally against --merge-base (origin/master) and pins the -baseline to it. Without a merge base build, or without a commit, the run is -compared against the latest master build, which may be newer than the -commit's base; the report warns when that happens. +computes it locally against --merge-base (upstream/master if it exists, else +origin/master) and pins the baseline to it. When that build does not exist, +the run is sent again without the pin. Without a merge base build, or without +a commit, the run is compared against the latest master build, which may be +newer than the commit's base; the report warns when that happens. --job-name (DASHBOARD_JOB_NAME) checks as a build of that pipeline instead, with its CI rules for the branch checked out here (--branch overrides it): the @@ -91,7 +92,10 @@ SHA_RE = re.compile(r"^[0-9a-f]{40}$") COMMIT_KEY = "run__flow__scripts_commit" -DEFAULT_MERGE_BASE_REF = "origin/master" +# Tried in order when --merge-base is not set. In a fork clone (gh repo fork +# --clone), origin is the fork, whose master can be far behind the project's, +# and upstream is the project. +DEFAULT_MERGE_BASE_REFS = ("upstream/master", "origin/master") # NAME_W must match the first field width in ROW_FMT. NAME_W = 46 @@ -273,27 +277,48 @@ def git(*args): return None +def default_merge_base_ref(): + """The first of DEFAULT_MERGE_BASE_REFS that exists here, else the last one.""" + for ref in DEFAULT_MERGE_BASE_REFS: + if git("rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"): + return ref + return DEFAULT_MERGE_BASE_REFS[-1] + + @functools.lru_cache(maxsize=None) def local_merge_base(commit, ref): - """Returns (sha, error) for pinning a commit that no remote has. + """Returns (sha, warning) for pinning a commit that no remote has. The dashboard finds a merge base through GitHub, so a commit that was never pushed falls back to the latest master build, which may be newer than the commit's base. Both are None when the dashboard can resolve the commit itself: it is on a remote branch, or it is not in this repository (a - downloaded CI artifact). error is set when the commit is local but has no - merge base with ref. + downloaded CI artifact). warning is set when the commit should be pinned + but cannot be; the commit is then sent unpinned, like a pushed one. """ if git("cat-file", "-e", f"{commit}^{{commit}}") is None: return None, None - if git("for-each-ref", "--count=1", "--contains", commit, "refs/remotes"): + # Empty output means no remote branch has the commit. None means git + # failed, so it is not known, and a pin could hide a valid baseline. + remotes = git("for-each-ref", "--count=1", "--contains", commit, "refs/remotes") + if remotes is None: + return None, ( + f"git could not tell whether {commit[:12]} is on a remote, so the " + "baseline was not pinned to its local merge base." + ) + if remotes: return None, None sha = git("merge-base", commit, ref) + if not sha and not git("rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"): + return None, ( + f"{ref} does not exist here, so the baseline of {commit[:12]} was not " + "pinned. Set DASHBOARD_MERGE_BASE to an existing ref, or to none." + ) if not sha: return None, ( - f"commit {commit[:12]} is on no remote and has no merge base with " - f"{ref}. Set DASHBOARD_MERGE_BASE to another ref, or to none to " - "compare against the latest master build." + f"{commit[:12]} is on no remote and has no merge base with {ref}, " + "so the baseline was not pinned. Set DASHBOARD_MERGE_BASE to a ref " + "that shares history with it." ) return sha, None @@ -366,9 +391,23 @@ def check_run( "error": None, "metric_count": 0, "dropped": [], + # How an automatic merge-base pin went: the pinned sha and its ref, the + # pin dropped because it had no build, or why no pin was made. "merge_base": None, + "merge_base_ref": None, + "unbuilt_merge_base": None, + "pin_warning": None, } + def ask(payload): + response, error = post_check(endpoint, api_key, payload) + # render_run reads the response as a mapping. A well-formed endpoint + # always sends one, but a proxy error page or a bare JSON list would + # not, so reject it here instead of raising a level up. + if error is None and not isinstance(response, dict): + error = f"unexpected response type: {type(response).__name__}" + return response, error + # One run's surprise must not take down the sweep: this is called from a # thread pool, so an escaping exception would surface out of pool.map and # skip the report for every other run. Report it as this run's error and let @@ -389,13 +428,14 @@ def check_run( commit = commit or recorded_commit result["commit"] = commit + automatic_pin = None if merge_base_ref and commit and not base_commit: - base_commit, error = local_merge_base(commit, merge_base_ref) - if error: - result["status"] = "invalid" - result["error"] = error - return result - result["merge_base"] = base_commit + automatic_pin, result["pin_warning"] = local_merge_base( + commit, merge_base_ref + ) + result["merge_base"] = automatic_pin + result["merge_base_ref"] = merge_base_ref + base_commit = automatic_pin # No JobName and no PipelineId makes this a checkout to the dashboard: # it picks the pipeline itself, the first of its default baseline @@ -415,18 +455,20 @@ def check_run( if base_commit: payload["BaseCommitSha"] = base_commit - response, error = post_check(endpoint, api_key, payload) + response, error = ask(payload) + # An automatic pin is a guess, and the merge base often has no build of + # the design. Ask again without it, so the dashboard falls back to the + # latest master build as it does for any commit. An explicit + # --base-commit stays exact. + if automatic_pin and error is None and response.get("status") == "no_baseline": + del payload["BaseCommitSha"] + result["merge_base"] = None + result["unbuilt_merge_base"] = automatic_pin + response, error = ask(payload) if error is not None: result["error"] = error return result - # render_run reads the response as a mapping. A well-formed endpoint - # always sends one, but a proxy error page or a bare JSON list would - # not, so reject it here instead of raising a level up. - if not isinstance(response, dict): - result["error"] = f"unexpected response type: {type(response).__name__}" - return result - result["response"] = response result["status"] = response.get("status", "unknown") except Exception as error: # reported as this run's error, never raised @@ -446,6 +488,31 @@ def dropped_lines(result): ] +def pin_lines(result): + """The warning that says how an automatic merge-base pin went, if one was tried. + + A warning, not information: the merge base comes from a local ref, and a + stale one silently moves the baseline. + """ + if result["pin_warning"]: + return [f"[WARN] {result['pin_warning']}"] + commit = (result.get("commit") or "")[:12] + ref = result["merge_base_ref"] + if result["unbuilt_merge_base"]: + return [ + f"[WARN] {commit} is on no remote, and its local merge base " + f"{result['unbuilt_merge_base'][:12]} with {ref} has no build of " + "this design, so the check was sent again without that pin." + ] + if result["merge_base"]: + return [ + f"[WARN] {commit} is on no remote, so the baseline was pinned to its " + f"local merge base {result['merge_base'][:12]} with {ref}. Set " + f"DASHBOARD_MERGE_BASE if {ref} is out of date." + ] + return [] + + def render_run(result, verbose): """Builds the report lines for one run.""" identity = f"{result['platform']}/{result['design']}/{result['variant']}" @@ -470,13 +537,7 @@ def render_run(result, verbose): f"[WARN] {identity}: no baseline to compare against " f"(strategy={strategy}). Nothing was checked." ) - if result["merge_base"]: - lines.append( - f"[WARN] The baseline was pinned to {result['merge_base'][:12]}, " - "the local merge base, which has no build of this design. Set " - "DASHBOARD_MERGE_BASE=none to compare against the latest master " - "build instead." - ) + lines.extend(pin_lines(result)) lines.extend(dropped_lines(result)) return lines @@ -497,11 +558,7 @@ def render_run(result, verbose): f"({(response.get('baseCommitSha') or '?')[:12]}, {strategy}" f"{', ' + pipeline if pipeline else ''})" ) - if result["merge_base"]: - lines.append( - f"[INFO] {result['commit'][:12]} is on no remote, so the baseline " - f"was pinned to its local merge base {result['merge_base'][:12]}" - ) + lines.extend(pin_lines(result)) # The latest master build may be newer than the commit's base, so upstream # changes since then would read as this run's regressions, and the result # moves with master. @@ -701,12 +758,12 @@ def parse_args(argv): ) parser.add_argument( "--merge-base", - default=os.environ.get("DASHBOARD_MERGE_BASE") or DEFAULT_MERGE_BASE_REF, + default=os.environ.get("DASHBOARD_MERGE_BASE") or None, metavar="REF", help="Pin a commit that is on no remote branch to its merge base with " - f"REF, since the dashboard cannot find one. Default: " - f"{DEFAULT_MERGE_BASE_REF}; none turns it off. Ignored with --base-commit " - "or --job-name. Env: DASHBOARD_MERGE_BASE.", + "REF, since the dashboard cannot find one. Default: upstream/master if " + "it exists, else origin/master; none turns it off. Ignored with " + "--base-commit or --job-name. Env: DASHBOARD_MERGE_BASE.", ) parser.add_argument( "--api-url", @@ -855,8 +912,9 @@ def run_check(argv=None): # A pipeline run follows its CI branch rules, and CI checkouts often fetch # no remote branches, so every commit there would look unpushed. merge_base_ref = None - if not (base_commit or args.job_name) and args.merge_base.lower() != "none": - merge_base_ref = args.merge_base + pin_off = (args.merge_base or "").lower() == "none" + if not (base_commit or args.job_name or pin_off): + merge_base_ref = args.merge_base or default_merge_base_ref() # Only a pipeline run uses the branch; a checkout is resolved from the commit. branch = "" if args.job_name: diff --git a/flow/util/utils.mk b/flow/util/utils.mk index 15c20df73d..c0549e7be7 100644 --- a/flow/util/utils.mk +++ b/flow/util/utils.mk @@ -31,8 +31,9 @@ metadata-generate: # commit it records (the master build at its merge base, from the first of # the dashboard's default pipelines that has the design), using the # dashboard's rule configs. An unpushed commit's merge base is computed -# locally against DASHBOARD_MERGE_BASE (default origin/master); without one -# the latest master build is used, with a warning. Set DASHBOARD_BASE_COMMIT +# locally against DASHBOARD_MERGE_BASE (default upstream/master if it exists, +# else origin/master); without a build there the latest master build is used, +# with a warning. Set DASHBOARD_BASE_COMMIT # to pin the baseline, DASHBOARD_API_KEY for a private platform and # DASHBOARD_JOB_NAME to check as a build of one pipeline instead. A failed # rule fails the target; an unreachable dashboard or a design without a