Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 19 additions & 13 deletions docs/contrib/Metrics.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,16 +57,21 @@ artifact, is compared against that commit's own build.
The dashboard finds the merge base through GitHub, so it cannot find one for a
commit that was never pushed. The check handles that commit itself: when no
remote branch in your checkout contains the commit, it computes
`git merge-base <commit> origin/master` locally and pins the baseline to that
build. A commit that is on a remote, or that is not in your checkout at all, is
left to the dashboard.

When the dashboard has no `master` build at the merge base, or no merge base,
it compares against the latest `master` build instead (the latest one before
the commit, if it knows the commit's date). That build may be newer than the
commit's base, so changes merged since then can show up as regressions or hide
real ones, and the result moves with `master`. The report warns whenever this
happens; set `DASHBOARD_BASE_COMMIT` to compare against an exact build instead.
`git merge-base <commit> upstream/master` locally (`origin/master` when there
is no `upstream` remote) and pins the baseline to that build. The report warns
with the merge base and the ref it came from, because an outdated ref moves
the baseline. A commit that is on a remote, or that is not in your checkout at
all, is left to the dashboard. So is a commit with no merge base with the ref,
with a warning.

When the pinned merge base has no build of the design, the check is sent again
without the pin. When the dashboard has no `master` build at the merge base, or
no merge base, it compares against the latest `master` build instead (the
latest one before the commit, if it knows the commit's date). That build may
be newer than the commit's base, so changes merged since then can show up as
regressions or hide real ones, and the result moves with `master`. The report
warns whenever this happens; set `DASHBOARD_BASE_COMMIT` to compare against an
exact build instead.

These environment variables shape the comparison:

Expand All @@ -85,9 +90,10 @@ These environment variables shape the comparison:
it unset for a local run, private or not. With it set, an unpushed commit is
not pinned to its local merge base.
- `DASHBOARD_MERGE_BASE` (`--merge-base`) names the ref an unpushed commit's
merge base is computed against, `origin/master` by default. Set it to your
upstream ref if `origin` is a fork with an outdated `master`, or to `none` to
leave unpushed commits to the dashboard. It does not apply together with
merge base is computed against. The default is `upstream/master` when that
ref exists, else `origin/master`. Set it if your project remote has another
name, or to `none` to leave unpushed commits to the dashboard. It does not
apply together with
`DASHBOARD_BASE_COMMIT` or `DASHBOARD_JOB_NAME`.
- `DASHBOARD_BASE_COMMIT` (`--base-commit`) pins the baseline to the build of
that commit. Nothing else is tried: if that commit has no build of the
Expand Down
113 changes: 100 additions & 13 deletions flow/test/test_checkQorMetrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -276,13 +276,70 @@ def test_commit_missing_from_the_checkout_is_left_to_the_dashboard(self):
_, _, payloads = self.check(COMMIT)
self.assertNotIn("BaseCommitSha", payloads[0])

def test_unknown_ref_is_an_error(self):
def test_pin_warns_with_the_ref(self):
repository, master, unpushed = self.repository()
self.enter(repository)
_, output, _ = self.check(unpushed)
self.assertIn(
f"[WARN] {unpushed[:12]} is on no remote, so the baseline was pinned "
f"to its local merge base {master[:12]} with origin/master",
output,
)

def test_upstream_master_is_the_default_when_it_exists(self):
# A fork clone: origin/master is the fork's old master, upstream/master
# the project's, and the local commit is on the project's.
repository, old_master, _ = self.repository()
git(repository, "reset", "-q", "--hard", old_master)
git(repository, "commit", "-q", "--allow-empty", "-m", "upstream")
upstream = git(repository, "rev-parse", "HEAD")
git(repository, "update-ref", "refs/remotes/upstream/master", upstream)
git(repository, "commit", "-q", "--allow-empty", "-m", "local")
unpushed = git(repository, "rev-parse", "HEAD")
self.enter(repository)
_, output, payloads = self.check(unpushed)
self.assertEqual(payloads[0]["BaseCommitSha"], upstream)
self.assertIn(f"{upstream[:12]} with upstream/master", output)

def test_no_merge_base_warns_and_sends_no_pin(self):
repository, _, unpushed = self.repository()
git(repository, "checkout", "-q", "--orphan", "unrelated")
git(repository, "commit", "-q", "--allow-empty", "-m", "unrelated")
unrelated = git(repository, "rev-parse", "HEAD")
git(repository, "update-ref", "refs/remotes/upstream/master", unrelated)
self.enter(repository)
code, output, payloads = self.check(unpushed, "--merge-base", "upstream/master")
self.assertEqual(code, 1)
self.assertEqual(payloads, [])
self.assertIn("has no merge base with upstream/master", output)
code, output, payloads = self.check(unpushed)
self.assertEqual(code, 0)
self.assertEqual(len(payloads), 1)
self.assertNotIn("BaseCommitSha", payloads[0])
self.assertIn(
"[WARN] " + unpushed[:12] + " is on no remote and has no merge base "
"with upstream/master, so the baseline was not pinned",
output,
)
self.assertIn("QoR check: PASS", output)

def test_missing_ref_warns_and_sends_no_pin(self):
repository, _, unpushed = self.repository()
self.enter(repository)
code, output, payloads = self.check(unpushed, "--merge-base", "upstrem/master")
self.assertEqual(code, 0)
self.assertNotIn("BaseCommitSha", payloads[0])
self.assertIn("[WARN] upstrem/master does not exist here", output)

def test_failed_remote_lookup_sends_no_pin(self):
repository, _, unpushed = self.repository()
self.enter(repository)
real_git = checkQorMetrics.git

def git_without_for_each_ref(*args):
return None if args[0] == "for-each-ref" else real_git(*args)

with patch.object(checkQorMetrics, "git", git_without_for_each_ref):
code, output, payloads = self.check(unpushed)
self.assertEqual(code, 0)
self.assertNotIn("BaseCommitSha", payloads[0])
self.assertIn(f"git could not tell whether {unpushed[:12]}", output)

def test_ref_from_the_environment(self):
repository, _, unpushed = self.repository()
Expand Down Expand Up @@ -310,15 +367,40 @@ def test_job_name_skips_it(self):
_, _, payloads = self.check(unpushed, "--job-name", "orfs-ci")
self.assertNotIn("BaseCommitSha", payloads[0])

def test_pinned_merge_base_without_a_build_suggests_turning_it_off(self):
def test_merge_base_without_a_build_is_sent_again_unpinned(self):
repository, master, unpushed = self.repository()
self.enter(repository)
code, output, payloads = self.run_check(
metadata=self.metadata(commit=unpushed),
dashboard=Dashboard(
response(status="no_baseline", strategy="none"),
response(strategy="latest-master"),
),
)
self.assertEqual(code, 0)
self.assertEqual(len(payloads), 2)
self.assertEqual(payloads[0]["BaseCommitSha"], master)
self.assertNotIn("BaseCommitSha", payloads[1])
self.assertIn(
f"merge base {master[:12]} with origin/master has no build of this "
"design, so the check was sent again without that pin",
output,
)
self.assertIn("compared against the latest master build", output)
self.assertIn("QoR check: PASS", output)

def test_explicit_base_commit_without_a_build_is_not_sent_again(self):
repository, _, unpushed = self.repository()
self.enter(repository)
code, output, _ = self.run_check(
code, output, payloads = self.run_check(
"--base-commit",
BASE,
metadata=self.metadata(commit=unpushed),
dashboard=Dashboard(response(status="no_baseline", strategy="none")),
)
self.assertEqual(code, 0)
self.assertIn("DASHBOARD_MERGE_BASE=none", output)
self.assertEqual(len(payloads), 1)
self.assertIn("QoR check: INCONCLUSIVE", output)


class MetricFilterTest(CheckQorMetricsTest):
Expand Down Expand Up @@ -411,17 +493,22 @@ def test_fail_lists_the_failed_rule(self):
failed = {
"metricName": "finish__timing__setup__ws",
"status": "fail",
"baselineValue": 2,
"currentValue": 1.5,
"limit": 1.9,
"baseValue": 2,
"targetValue": 1.5,
"thresholdValue": 1.9,
"deltaPct": -25,
}
code, output, _ = self.run_check(
dashboard=Dashboard(response(status="fail", failed=1, metrics=[failed]))
)
self.assertEqual(code, 1)
self.assertIn("finish__timing__setup__ws", output)
self.assertIn("-25.00", output)
self.assertIn(
checkQorMetrics.ROW_FMT.format(
"finish__timing__setup__ws", "2", "1.5000", "1.9000", "-25.00"
),
output,
)
self.assertIn("finish__timing__setup__ws 1.5 (base 2, limit 1.9)", output)
self.assertIn("QoR check: FAIL", output)

def test_latest_master_warns_with_a_commit(self):
Expand Down
Loading