HTML diff report: stream to disk, cap rows per node pair

ace.sample.yaml

@@ -22,6 +22,7 @@ postgres:
 table_diff:
   concurrency_factor: 0.5
   max_diff_rows: 1000000
+  max_html_rows: 10000         # max rows per node pair in the HTML report
   min_diff_block_size: 1
   max_diff_block_size: 1000000
   diff_block_size: 1000
@@ -52,6 +53,8 @@ mtree:
     max_block_size: 1000000
     # Max differing rows collected per node pair; 0 or absent means unbounded.
     max_diff_rows: 1000000
+    # Max rows per node pair in the HTML report (--output html).
+    max_html_rows: 10000
 
 # Example scheduler configuration:
 # schedule_jobs:

docs/CHANGELOG.md

@@ -5,6 +5,30 @@ All notable changes to ACE will be captured in this document. This project follo
 ## [Unreleased]
 
 ### Fixed
+- **`table-diff --output html` was killed by the OOM killer on large diffs.**
+  The HTML writer built the whole report in memory: a copy of the diff as
+  JSON, the markup of every row, and the final document in one buffer. With
+  several kilobytes of markup per row, a diff of 493,200 rows used about 4 GB
+  before the process was killed.
+  The writer now streams the report to disk one row at a time. It also shows
+  at most `max_html_rows` rows for each node pair (default `10000`, set in
+  `table_diff` and `mtree.diff` in `ace.yaml`, or with `--max-html-rows`),
+  because a report with hundreds of thousands of rows is too large for a
+  browser anyway. A truncated report says so at the top and in each node pair
+  section. A repair plan built in it has rules only for the rows shown and
+  `default_action: skip`, so `table-repair` does not change the rows that the
+  report did not show; the YAML starts with a comment that says this. With
+  three or more nodes this holds per key only: a plan rule is not tied to a
+  node pair, so it also acts on a key that the report hides in another pair.
+  The JSON report is not changed and always contains every row.
+- **HTML report: repair plans named the wrong rows for some primary keys.**
+  The page script read keys as JavaScript numbers, so a bigint above 2^53
+  could name the neighbouring row, and a text key such as `"007"` became the
+  number 7 and matched nothing. The plan now uses the keys exactly as the
+  diff file has them.
+- **HTML report: rows with primary keys that mix numbers and text were sorted
+  in a different order on each run.** The key comparison now puts numbers
+  before text, so the order is stable.
 - **Spock `add_node` failed on clusters where `mtree init` had been run.**
   `mtree init` created a `#` operator on `bytea_xor` without a schema name, so
   it landed in `public` while the function stayed in the ACE schema. Spock's

docs/commands/diff/table-diff.md

@@ -20,6 +20,7 @@ This command compares the data in the specified table across nodes in a cluster
 | `--concurrency-factor <float>` | `-c` | CPU ratio for concurrency (0.0–4.0, e.g. 0.5 uses half of available CPUs). Default `0.5`. |
 | `--compare-unit-size <int>` | `-u` | Recursive split size for mismatched blocks. Default `10000`. |
 | `--output <json\|html>` | `-o` | Report format. Default `json`. When `html`, both JSON and HTML files share the same timestamped prefix. |
+| `--max-html-rows <int>` |  | Maximum number of rows for each node pair in the HTML report. Default `0`, which means `table_diff.max_html_rows` from `ace.yaml`, or `10000` when that is not set either. There is no setting for "no limit"; set a large number instead. The JSON report always contains every row. |
 | `--nodes <list>` | `-n` | Comma-separated node list or `all`. Up to three-way diffs are supported. |
 | `--table-filter <WHERE>` | `-F` | Optional SQL `WHERE` clause applied on every node before hashing. |
 | `--against-origin <node>` |  | Limit the diff to rows whose `node_origin` matches this Spock node id or name (useful for failed-node recovery). |
@@ -119,6 +120,25 @@ ace table-diff my-cluster public._events \
       summary as both the raw filter and the effective filter (which also
       includes `--against-origin`/`--until` if set).
 3. Prefer `--output html` when you'll manually review diffs.
+    - The HTML report shows at most `max_html_rows` rows for each node pair
+      (default `10000`). A row is one primary key: a value difference or a
+      row missing on one node. Each row takes several kilobytes of markup, so
+      a report with hundreds of thousands of rows is too large for a browser.
+      When the diff has more rows, the report says so at the top and in each
+      node pair section, and the full list stays in the JSON file. Rows are
+      taken in report order: value differences, then rows missing on the
+      second node, then rows missing on the first node.
+    - A repair plan that you build in a truncated report has rules only for
+      the rows shown, and its `default_action` is `skip`. So when you run
+      `table-repair` with that plan and the full diff file, the rows that the
+      report did not show are not changed. The downloaded YAML starts with a
+      comment that says this. To repair every row, run `table-diff` again with
+      a larger `--max-html-rows`, or write the plan by hand.
+    - Known limit: a plan rule matches a primary key and a kind of
+      difference, not a node pair. With three or more nodes, a rule for a
+      key shown in one pair also acts on the same key in another pair, even
+      where the report hides it there. Check such keys before you run
+      `table-repair`.
 4. Use `--override-block-size` sparingly; the guardrails in `ace.yaml` prevent
    allocations that can overwhelm memory.
 5. The `max_diff_rows` setting caps the number of differing rows that

docs/commands/mtree/mtree-table-diff.md

@@ -31,6 +31,7 @@ holding the node's slot, then re-run, if you need a guaranteed-current drain.
 | `--nodes` | `-n` | Nodes to include (comma or `all`) | `all` |
 | `--max-cpu-ratio` | `-m` | Max CPU ratio | `0.5` |
 | `--output` | `-o` | `json` or `html` | `json` |
+| `--max-html-rows` |  | Max rows per node pair in the HTML report (`0` = use `mtree.diff.max_html_rows`, or `10000`) | `0` |
 | `--skip-cdc` | `-U` | Skip CDC processing (only rehash and compare) | `false` |
 | `--cdc-timeout` |  | Seconds to drain CDC before giving up (`0` = use `cdc_processing_timeout` / default) | `0` |
 | `--quiet` | `-q` | Suppress output | `false` |
@@ -45,6 +46,13 @@ holding the node's slot, then re-run, if you need a guaranteed-current drain.
 **Notes**
 
 - With `--output html`, both JSON and HTML reports are generated with matching timestamps.
+- The HTML report shows at most `max_html_rows` rows for each node pair
+  (`mtree.diff.max_html_rows`, default `10000`). The JSON report always
+  contains every row. When the HTML report is truncated, it says so, and a
+  repair plan built in it has rules only for the rows shown and
+  `default_action: skip`, so `table-repair` does not change the other rows.
+  With three or more nodes this holds per key, not per node pair: see the
+  known limit in the `table-diff` docs.
 - The number of differing rows collected per node pair is bounded by
   `mtree.diff.max_diff_rows` (the shipped `ace.yaml` sets `1000000`; if the key
   is absent or `0`, the diff is unbounded). When the cap is reached, enumeration

internal/cli/cli.go

@@ -142,6 +142,11 @@ func SetupCLI(version string) *cli.Command {
 
 	tableDiffFlags := append(commonFlags, diffFlags...)
 	tableDiffFlags = append(tableDiffFlags,
+		&cli.Int64Flag{
+			Name:  "max-html-rows",
+			Usage: "Max rows per node pair in the HTML report (0 = use max_html_rows from config, default 10000)",
+			Value: 0,
+		},
 		&cli.StringFlag{
 			Name:    "table-filter",
 			Aliases: []string{"F"},
@@ -376,6 +381,11 @@ func SetupCLI(version string) *cli.Command {
 	mtreeUpdateFlags = append(mtreeUpdateFlags, commonFlags...)
 
 	mtreeDiffFlags := []cli.Flag{
+		&cli.Int64Flag{
+			Name:  "max-html-rows",
+			Usage: "Max rows per node pair in the HTML report (0 = use max_html_rows from config, default 10000)",
+			Value: 0,
+		},
 		&cli.Float64Flag{
 			Name:    "max-cpu-ratio",
 			Aliases: []string{"m"},
@@ -948,6 +958,7 @@ func TableDiffCLI(cmd *cli.Command) error {
 	task.MaxConnections = cmd.Int("max-connections")
 	task.CompareUnitSize = cmd.Int("compare-unit-size")
 	task.Output = strings.ToLower(cmd.String("output"))
+	task.MaxHTMLRows = cmd.Int64("max-html-rows")
 	task.Nodes = cmd.String("nodes")
 	task.EnsurePgcrypto = cmd.Bool("ensure-pgcrypto")
 	scheduleEnabled := cmd.Bool("schedule")
@@ -1205,6 +1216,7 @@ func MtreeDiffCLI(cmd *cli.Command) error {
 	task.QuietMode = cmd.Bool("quiet")
 	task.MaxCpuRatio = cmd.Float64("max-cpu-ratio")
 	task.Output = cmd.String("output")
+	task.MaxHTMLRows = cmd.Int64("max-html-rows")
 	task.NoCDC = cmd.Bool("skip-cdc")
 	task.CDCTimeoutSec = cmd.Int("cdc-timeout")
 	task.Until = cmd.String("until")

internal/cli/default_config.yaml

@@ -22,6 +22,7 @@ postgres:
 table_diff:
   concurrency_factor: 0.5
   max_diff_rows: 1000000
+  max_html_rows: 10000         # max rows per node pair in the HTML report
   min_diff_block_size: 1
   max_diff_block_size: 1000000
   diff_block_size: 1000
@@ -52,6 +53,8 @@ mtree:
     max_block_size: 1000000
     # Max differing rows collected per node pair; 0 or absent means unbounded.
     max_diff_rows: 1000000
+    # Max rows per node pair in the HTML report (--output html).
+    max_html_rows: 10000
 
 # Example scheduler configuration:
 # schedule_jobs:

internal/consistency/diff/table_diff.go

@@ -105,6 +105,10 @@ type TableDiffTask struct {
 
 	CompareUnitSize int
 	MaxDiffRows     int64
+	// MaxHTMLRows limits how many entries the HTML report shows for each
+	// node pair. 0 means table_diff.max_html_rows from the config, and if
+	// that is not set either, utils.DefaultMaxHTMLRows.
+	MaxHTMLRows int64
 
 	DiffResult types.DiffOutput
 	diffMutex  sync.Mutex
@@ -680,6 +684,16 @@ func (t *TableDiffTask) Validate() error {
 		t.MaxDiffRows = cfg.TableDiff.MaxDiffRows
 	}
 
+	if t.MaxHTMLRows < 0 {
+		return fmt.Errorf("max_html_rows must be >= 0")
+	}
+	if cfg.TableDiff.MaxHTMLRows < 0 {
+		return fmt.Errorf("table_diff.max_html_rows in the config must be >= 0, got %d", cfg.TableDiff.MaxHTMLRows)
+	}
+	if t.MaxHTMLRows == 0 && cfg.TableDiff.MaxHTMLRows > 0 {
+		t.MaxHTMLRows = cfg.TableDiff.MaxHTMLRows
+	}
+
 	if t.ConcurrencyFactor > 4.0 || t.ConcurrencyFactor <= 0 {
 		return fmt.Errorf("invalid value range for concurrency_factor, must be > 0 and <= 4.0")
 	}
@@ -1053,6 +1067,7 @@ func (t *TableDiffTask) CloneForSchedule(ctx context.Context) *TableDiffTask {
 	cloned.InvokeMethod = t.InvokeMethod
 	cloned.CompareUnitSize = t.CompareUnitSize
 	cloned.MaxDiffRows = t.MaxDiffRows
+	cloned.MaxHTMLRows = t.MaxHTMLRows
 	cloned.EnsurePgcrypto = t.EnsurePgcrypto
 	cloned.AgainstOrigin = t.AgainstOrigin
 	cloned.Until = t.Until
@@ -1641,7 +1656,7 @@ func (t *TableDiffTask) ExecuteTask() (err error) {
 
 	t.AddPrimaryKeyToDiffSummary()
 
-	jsonPath, _, err := utils.WriteDiffReport(t.DiffResult, t.Schema, t.BaseTable, t.Output)
+	jsonPath, _, err := utils.WriteDiffReport(t.DiffResult, t.Schema, t.BaseTable, t.Output, t.MaxHTMLRows)
 	if err != nil {
 		return err
 	}

internal/consistency/diff/table_diff_html_rows_test.go

@@ -0,0 +1,69 @@
+// ///////////////////////////////////////////////////////////////////////////
+//
+// # ACE - Active Consistency Engine
+//
+// Copyright (C) 2023 - 2026, pgEdge (https://www.pgedge.com/)
+//
+// This software is released under the PostgreSQL License:
+// https://opensource.org/license/postgresql
+//
+// ///////////////////////////////////////////////////////////////////////////
+
+package diff
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/pgedge/ace/pkg/config"
+)
+
+// TestValidateMaxHTMLRows checks how Validate resolves max_html_rows: the
+// task value wins, 0 falls back to table_diff.max_html_rows, and a negative
+// value in either place is an error. Validate goes on to fail on other
+// checks later (the task here is incomplete), so only the max_html_rows
+// errors and the resolved value are checked.
+func TestValidateMaxHTMLRows(t *testing.T) {
+	for _, tc := range []struct {
+		name     string
+		task     int64
+		cfg      int64
+		want     int64
+		wantErrs string
+	}{
+		{name: "task value wins", task: 7, cfg: 500, want: 7},
+		{name: "config when task is 0", task: 0, cfg: 500, want: 500},
+		{name: "both 0 leave the default to the writer", task: 0, cfg: 0, want: 0},
+		{name: "negative task value", task: -1, cfg: 500, wantErrs: "max_html_rows must be >= 0"},
+		{name: "negative config value", task: 0, cfg: -5, wantErrs: "table_diff.max_html_rows in the config must be >= 0"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := &config.Config{}
+			cfg.TableDiff.MinBlockSize = 1
+			cfg.TableDiff.MaxBlockSize = 1000000
+			cfg.TableDiff.MaxHTMLRows = tc.cfg
+			config.Set(cfg)
+			t.Cleanup(func() { config.Set(nil) })
+
+			task := NewTableDiffTask()
+			task.ClusterName = "c"
+			task.QualifiedTableName = "public.t"
+			task.BlockSize = 1000
+			task.MaxHTMLRows = tc.task
+
+			err := task.Validate()
+			if tc.wantErrs != "" {
+				if err == nil || !strings.Contains(err.Error(), tc.wantErrs) {
+					t.Fatalf("Validate: got error %v, want one containing %q", err, tc.wantErrs)
+				}
+				return
+			}
+			if err != nil && strings.Contains(err.Error(), "max_html_rows") {
+				t.Fatalf("Validate: unexpected max_html_rows error: %v", err)
+			}
+			if task.MaxHTMLRows != tc.want {
+				t.Errorf("MaxHTMLRows after Validate: got %d, want %d", task.MaxHTMLRows, tc.want)
+			}
+		})
+	}
+}

internal/consistency/mtree/merkle.go

@@ -110,6 +110,11 @@ type MerkleTreeTask struct {
 	// and OOM the process.
 	MaxDiffRows int64
 
+	// MaxHTMLRows limits how many entries the HTML report shows for each
+	// node pair. 0 means mtree.diff.max_html_rows from the config, and if
+	// that is not set either, utils.DefaultMaxHTMLRows.
+	MaxHTMLRows int64
+
 	DiffResult     types.DiffOutput
 	diffMutex      sync.Mutex
 	diffRowKeySets map[string]map[string]map[string]struct{}
@@ -2401,6 +2406,12 @@ func (m *MerkleTreeTask) DiffMtree() (err error) {
 	if m.MaxDiffRows < 0 {
 		return fmt.Errorf("max_diff_rows must be >= 0, got %d", m.MaxDiffRows)
 	}
+	if m.MaxHTMLRows < 0 {
+		return fmt.Errorf("max_html_rows must be >= 0, got %d", m.MaxHTMLRows)
+	}
+	if cfg := config.Get(); cfg != nil && cfg.MTree.Diff.MaxHTMLRows < 0 {
+		return fmt.Errorf("mtree.diff.max_html_rows in the config must be >= 0, got %d", cfg.MTree.Diff.MaxHTMLRows)
+	}
 
 	if err = m.UpdateMtree(true); err != nil {
 		// A missing tree already carries a complete, actionable message;
@@ -2443,6 +2454,11 @@ func (m *MerkleTreeTask) DiffMtree() (err error) {
 			m.MaxDiffRows = cfg.MTree.Diff.MaxDiffRows
 		}
 	}
+	if m.MaxHTMLRows == 0 {
+		if cfg := config.Get(); cfg != nil && cfg.MTree.Diff.MaxHTMLRows > 0 {
+			m.MaxHTMLRows = cfg.MTree.Diff.MaxHTMLRows
+		}
+	}
 	m.diffRowCounts = make(map[string]int64)
 	m.diffLimitWarned = false
 	m.pairCompareErrs = make(map[string]bool)
@@ -2615,7 +2631,7 @@ func (m *MerkleTreeTask) DiffMtree() (err error) {
 			logger.Warn("mtree table-diff stopped after reaching max_diff_rows=%d; additional differences may exist", m.MaxDiffRows)
 		}
 
-		diffPath, _, writeErr := utils.WriteDiffReport(m.DiffResult, m.Schema, m.Table, m.Output)
+		diffPath, _, writeErr := utils.WriteDiffReport(m.DiffResult, m.Schema, m.Table, m.Output, m.MaxHTMLRows)
 		if writeErr != nil {
 			return writeErr
 		}