From 434028a816158af7a99c9098f501d7de76f620bf Mon Sep 17 00:00:00 2001 From: Andrei Lepikhov Date: Thu, 24 Sep 2026 15:48:33 +0200 Subject: [PATCH 1/7] Make the key order of the HTML diff report total comparePKComponent compared two numbers by value, but a number and a string as strings. That order is not transitive: "1a" < "9" < "10", but "10" < "1a". The keys come from map iteration, so sort.Slice put rows with such keys in a different order on each run. Numbers now come before all other strings. NaN and the infinities count as text, because NaN is not equal even to itself. --- pkg/common/html_reporter.go | 21 +++++++++++------ pkg/common/html_reporter_test.go | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 pkg/common/html_reporter_test.go diff --git a/pkg/common/html_reporter.go b/pkg/common/html_reporter.go index 131b280..e6e5fb6 100644 --- a/pkg/common/html_reporter.go +++ b/pkg/common/html_reporter.go @@ -17,6 +17,7 @@ import ( "encoding/json" "fmt" "html/template" + "math" "path/filepath" "sort" "strconv" @@ -726,10 +727,16 @@ func comparePKKey(a, b string) int { } } +// comparePKComponent orders numbers by value and before all other strings, +// and other strings byte-wise. The order must be total. The keys come from +// map iteration, and with an order that is not transitive (as when a number +// and a string compared as strings, so that "1a" < "9" < "10" but +// "10" < "1a") sort.Slice put the rows in a different order on each run. func comparePKComponent(a, b string) int { numA, okA := parseNumeric(a) numB, okB := parseNumeric(b) - if okA && okB { + switch { + case okA && okB: switch { case numA < numB: return -1 @@ -738,14 +745,12 @@ func comparePKComponent(a, b string) int { default: return 0 } - } - if a < b { + case okA: return -1 - } - if a > b { + case okB: return 1 } - return 0 + return strings.Compare(a, b) } func parseNumeric(val string) (float64, bool) { @@ -753,7 +758,9 @@ func parseNumeric(val string) (float64, bool) { return 0, false } num, err := strconv.ParseFloat(val, 64) - if err != nil { + // NaN is not equal to anything, itself included, and would break the + // order; treat it and the infinities as text. + if err != nil || math.IsNaN(num) || math.IsInf(num, 0) { return 0, false } return num, true diff --git a/pkg/common/html_reporter_test.go b/pkg/common/html_reporter_test.go new file mode 100644 index 0000000..a37db17 --- /dev/null +++ b/pkg/common/html_reporter_test.go @@ -0,0 +1,39 @@ +// /////////////////////////////////////////////////////////////////////////// +// +// # ACE - Active Consistency Engine +// +// Copyright (C) 2023 - 2026, pgEdge (https://www.pgedge.com/) +// +// This software is released under the PostgreSQL License: +// https://opensource.org/license/postgresql +// +// /////////////////////////////////////////////////////////////////////////// + +package common + +import "testing" + +func TestComparePKComponentIsATotalOrder(t *testing.T) { + vals := []string{"1", "1.0", "2", "9", "10", "1a", "10a", "9a", "", "NaN", "Inf", "-1", "abc", "a"} + sign := func(x int) int { + switch { + case x < 0: + return -1 + case x > 0: + return 1 + } + return 0 + } + for _, a := range vals { + for _, b := range vals { + if sign(comparePKComponent(a, b)) != -sign(comparePKComponent(b, a)) { + t.Errorf("not antisymmetric: %q, %q", a, b) + } + for _, c := range vals { + if comparePKComponent(a, b) < 0 && comparePKComponent(b, c) < 0 && comparePKComponent(a, c) >= 0 { + t.Errorf("not transitive: %q < %q < %q but not %q < %q", a, b, c, a, c) + } + } + } + } +} From 4106731ba6f89a17969139b42fc307b409d886fa Mon Sep 17 00:00:00 2001 From: Andrei Lepikhov Date: Thu, 24 Sep 2026 15:51:03 +0200 Subject: [PATCH 2/7] Stream the HTML diff report to disk table-diff --output html was killed by the OOM killer on a diff of 493,200 rows, while the JSON report for the same diff was written without trouble. The HTML writer held the whole report in memory at once: a json.Marshal copy of the full diff for the page script, a struct with escaped cells for every row, and the rendered page in one bytes.Buffer. Each row takes several kilobytes of markup, so the buffer alone grew to gigabytes. The template is now split into blocks: page head, pair head, one block per row, pair tail and page tail. The writer runs them one by one into a buffered writer on the report file, and builds the data of a row only while that row is written. Only the row keys of the whole diff stay in memory. If writing fails, the half-written file is removed, and the error names the JSON report, which is complete. The page script no longer gets a copy of the whole diff. It gets one entry per row, with the pair, the kind of difference, and each primary key value as the JSON text of the diff file, and it builds the repair plan from that. The script copies the key text into the YAML as it is. Before, it read keys as JavaScript numbers, so a bigint above 2^53 could name the next row, and a text key such as "007" became 7 and matched no row. Range rules are used only when every key of the diff is a whole number. On a synthetic diff of 493,200 rows, peak memory fell from 4.8 GB to 0.55 GB; the diff itself takes 0.2 GB. The markup of the page does not change. Tests: the rows on the page match the embedded rows for each pair; key literals; the output is the same from run to run; a write error in each part of the page is returned and leaves no file. Three tests run the page script in Node.js (they skip without node) and resolve the plan with the repair executor. --- .../consistency/repair/html_plan_e2e_test.go | 201 +++++ pkg/common/html_reporter.go | 730 ++++++++++++------ pkg/common/html_reporter_test.go | 382 ++++++++- pkg/common/templates/diff_report.html | 121 +-- pkg/common/templates/diff_report.js | 202 ++--- pkg/common/utils.go | 3 +- 6 files changed, 1209 insertions(+), 430 deletions(-) create mode 100644 internal/consistency/repair/html_plan_e2e_test.go diff --git a/internal/consistency/repair/html_plan_e2e_test.go b/internal/consistency/repair/html_plan_e2e_test.go new file mode 100644 index 0000000..974776d --- /dev/null +++ b/internal/consistency/repair/html_plan_e2e_test.go @@ -0,0 +1,201 @@ +// /////////////////////////////////////////////////////////////////////////// +// +// # ACE - Active Consistency Engine +// +// Copyright (C) 2023 - 2026, pgEdge (https://www.pgedge.com/) +// +// This software is released under the PostgreSQL License: +// https://opensource.org/license/postgresql +// +// /////////////////////////////////////////////////////////////////////////// + +package repair + +import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + planner "github.com/pgedge/ace/internal/consistency/repair/plan" + utils "github.com/pgedge/ace/pkg/common" + "github.com/pgedge/ace/pkg/types" +) + +// These tests check the whole path of a repair plan built in the HTML +// report: table-diff writes the JSON and HTML reports, the page script +// (run in Node.js, with no user choices) builds the YAML plan, and the +// repair executor resolves that plan against the full JSON diff. They skip +// when node is not installed. + +// planScript runs buildPlanYaml from the page on the page's embedded data. +// It takes the script and the data out of the page itself, so it tests what +// a browser would run. +const planScript = ` +const fs = require('fs'); +const [htmlPath, outPath] = process.argv.slice(2); +const page = fs.readFileSync(htmlPath, 'utf8'); +const data = page.match(/\n") + return err } func highlightDifference(a, b string) (template.HTML, template.HTML) { @@ -458,8 +697,9 @@ func buildDiffBreakdown(diffCounts map[string]int) []htmlPairCount { } // buildRowKey returns the identity a row is matched by across the two nodes. -// It falls back to the row's position when there is no usable primary key, -// which pairs nothing but at least keeps distinct rows distinct. +// It falls back to the row's position when there is no usable primary key. +// That keeps distinct rows of one node distinct, but it pairs rows of the two +// nodes by position only: __row_0 on A meets __row_0 on B, whatever they hold. func buildRowKey(row types.OrderedMap, primaryKey []string, index int) string { if len(primaryKey) == 0 { return fmt.Sprintf("__row_%d", index) diff --git a/pkg/common/html_reporter_test.go b/pkg/common/html_reporter_test.go index a37db17..ea0d83c 100644 --- a/pkg/common/html_reporter_test.go +++ b/pkg/common/html_reporter_test.go @@ -11,7 +11,327 @@ package common -import "testing" +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/pgedge/ace/pkg/types" +) + +// htmlTestDiff builds a two-node diff with the given number of value +// differences, rows missing on n2 and rows missing on n1. Primary keys are +// 1..N in that order, so the report order is easy to predict. +func htmlTestDiff(valueDiffs, missingOnB, missingOnA int, note string) types.DiffOutput { + row := func(id int, val string) types.OrderedMap { + return types.OrderedMap{ + {Key: "id", Value: id}, + {Key: "val", Value: val}, + {Key: "note", Value: note}, + } + } + var a, b []types.OrderedMap + id := 0 + for i := 0; i < valueDiffs; i++ { + id++ + a = append(a, row(id, "on-n1")) + b = append(b, row(id, "on-n2")) + } + for i := 0; i < missingOnB; i++ { + id++ + a = append(a, row(id, "only-n1")) + } + for i := 0; i < missingOnA; i++ { + id++ + b = append(b, row(id, "only-n2")) + } + return types.DiffOutput{ + NodeDiffs: map[string]types.DiffByNodePair{ + "n1/n2": {Rows: map[string][]types.OrderedMap{"n1": a, "n2": b}}, + }, + Summary: types.DiffSummary{ + Schema: "public", + Table: "t", + Nodes: []string{"n1", "n2"}, + PrimaryKey: []string{"id"}, + DiffRowsCount: map[string]int{"n1/n2": valueDiffs + missingOnB + missingOnA}, + }, + } +} + +type htmlTestData struct { + Summary types.DiffSummary `json:"summary"` + HTMLReport htmlReportInfo `json:"html_report"` + Rows []htmlPlanRow `json:"rows"` +} + +// typeCounts counts the embedded plan rows of each type for one pair. +func (d htmlTestData) typeCounts(pair string) map[string]int { + c := map[string]int{} + for _, r := range d.Rows { + if r.Pair == pair { + c[r.Type]++ + } + } + return c +} + +var diffDataRe = regexp.MustCompile(`(?s)`) + +// renderHTMLTestReport writes the report and returns the page and the parsed +// embedded diff data. +func renderHTMLTestReport(t *testing.T, diff types.DiffOutput) (string, htmlTestData) { + t.Helper() + jsonPath := filepath.Join(t.TempDir(), "public_t_diffs-20260101000000.json") + htmlPath, err := writeHTMLDiffReport(diff, jsonPath) + if err != nil { + t.Fatalf("writeHTMLDiffReport: %v", err) + } + raw, err := os.ReadFile(htmlPath) + if err != nil { + t.Fatalf("read report: %v", err) + } + page := string(raw) + if !strings.HasSuffix(strings.TrimSpace(page), "") { + t.Fatal("report does not end with ") + } + + m := diffDataRe.FindStringSubmatch(page) + if m == nil { + t.Fatal("report has no diff-data script") + } + var data htmlTestData + if err := json.Unmarshal([]byte(m[1]), &data); err != nil { + t.Fatalf("embedded diff data is not valid JSON: %v", err) + } + return page, data +} + +// countRows counts rendered rows of one type ("value_diff", "missing_in_a", +// "missing_in_b"). Every rendered row has exactly one select checkbox. +func countRows(page, rowType string) int { + re := regexp.MustCompile(`class="row-select" data-pk="[^"]*" data-type="` + rowType + `"`) + return len(re.FindAllStringIndex(page, -1)) +} + +func TestHTMLReportRowsAndData(t *testing.T) { + page, data := renderHTMLTestReport(t, htmlTestDiff(4, 2, 1, "x")) + + if !strings.Contains(page, ">7 entries<") { + t.Error("section pill does not show the entry count") + } + if got := countRows(page, "value_diff"); got != 4 { + t.Errorf("value rows: got %d, want 4", got) + } + if got := countRows(page, "missing_in_b"); got != 2 { + t.Errorf("missing_in_b rows: got %d, want 2", got) + } + if got := countRows(page, "missing_in_a"); got != 1 { + t.Errorf("missing_in_a rows: got %d, want 1", got) + } + + want := map[string]int{"row_mismatch": 4, "missing_on_n2": 2, "missing_on_n1": 1} + if got := data.typeCounts("n1/n2"); fmt.Sprint(got) != fmt.Sprint(want) { + t.Errorf("embedded rows: got %v, want %v", got, want) + } + if data.Summary.Table != "t" || len(data.Summary.PrimaryKey) != 1 { + t.Errorf("embedded summary is wrong: %+v", data.Summary) + } +} + +// Primary key values go into the embedded JSON inside a