diff --git a/build/app.zig b/build/app.zig index ed7c88017..e4405e963 100644 --- a/build/app.zig +++ b/build/app.zig @@ -841,7 +841,7 @@ pub fn construct(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std. // A document state as the bytes that differ from another: how a track's checkpoints // stay small. std-only (see state_delta.zig); `Tracks` keeps its checkpoints with it. .{ "fizzy-state-delta-tests", "app/state_delta.zig" }, - // Image diff: RGBA8 rows and tiles to the rects that differ. std-only (see + // Image diff (RGBA8 to rects) and text diff (histogram lines, unified output). std-only (see // core/diff.zig); `bench-diff` measures it. .{ "fizzy-diff-tests", "core/diff.zig" }, // When a document's transport shows and for how long, how far a held undo goes, which @@ -1507,7 +1507,7 @@ pub fn construct(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std. bench_step.dependOn(&run_bench.step); } - // `zig build bench-diff` — core.diff on a 4K image and a 2048x2048 sprite of 50 layers (prints + // `zig build bench-diff` — core.diff on a 4K image, a 2048x2048 sprite of 50 layers and 1 MB of text (prints // timings). Same rules as `bench-tape`; std-only, so it needs no window. { const bench_module = b.createModule(.{ @@ -1522,7 +1522,7 @@ pub fn construct(b: *std.Build, target: std.Build.ResolvedTarget, optimize: std. })); const bench_diff = b.addTest(.{ .name = "fizzy-bench-diff", .root_module = bench_module }); - const bench_step = b.step("bench-diff", "Benchmark core.diff: a 4K image, a 50-layer sprite (prints timings)"); + const bench_step = b.step("bench-diff", "Benchmark core.diff: a 4K image, a 50-layer sprite, 1 MB of text (prints timings)"); const run_bench = b.addRunArtifact(bench_diff); run_bench.has_side_effects = true; bench_step.dependOn(&run_bench.step); diff --git a/core/core.zig b/core/core.zig index ba3c86295..103db04e8 100644 --- a/core/core.zig +++ b/core/core.zig @@ -106,7 +106,7 @@ pub const lsp = @import("lsp/lsp.zig"); /// lower scores are better. pub const fuzzy = @import("fuzzy.zig"); -/// Std-only diffing (images so far): which regions of a document's state differ. See `diff.zig`. +/// Std-only diffing (images and text): which regions of a document's state differ. See `diff.zig`. pub const diff = @import("diff.zig"); /// The project's file set: one cached, searchable view of what is on disk, owned by the host and diff --git a/core/diff.zig b/core/diff.zig index 8cd039f09..3bdb9130b 100644 --- a/core/diff.zig +++ b/core/diff.zig @@ -6,8 +6,15 @@ pub const Image = @import("diff/Image.zig"); /// A keepable per-tile hash of an image, 8 bytes per tile, to diff a past state against. pub const TileGrid = @import("diff/TileGrid.zig"); +/// A line diff of two texts: histogram with a Myers fallback, over a reusable scratch. +pub const Text = @import("diff/Text.zig"); +/// A text diff's hunks as a standard unified diff, with a size cap. +pub const unified = @import("diff/unified.zig"); test { _ = Image; _ = TileGrid; + _ = Text; + _ = unified; + _ = @import("diff/tests.zig"); } diff --git a/core/diff/Text.zig b/core/diff/Text.zig new file mode 100644 index 000000000..40b211afd --- /dev/null +++ b/core/diff/Text.zig @@ -0,0 +1,498 @@ +//! A line diff of two texts: git's histogram algorithm over interned line ids, Myers (linear +//! space, bounded cost) where histogram finds no anchor, and the hunks slid to where they read +//! best. +//! +//! This file is the reusable scratch: keep one `Text` per caller, call `diff` as often as +//! needed, and nothing allocates once its buffers have grown to the biggest input seen. The +//! hunks `diff` returns live in the scratch until the next call. +//! +//! Lines end after `\n` and keep it, so `\r\n` is part of the line and a last line without a +//! newline differs from the same line with one. Binary input is just long lines. +//! +//! std-only, so plugins, the web and a plain test root all use it (`plans/DIFF_PLAN.md` 2.1-2.2). +const std = @import("std"); + +const Text = @This(); + +/// One changed span: `a`'s lines `a_start..a_start + a_len` became `b`'s lines +/// `b_start..b_start + b_len`. Either length may be zero (a pure insert or delete). The byte +/// ranges are the same spans in the texts, whole lines including their `\n`. +pub const Hunk = struct { + a_start: u32, + a_len: u32, + b_start: u32, + b_len: u32, + a_byte: usize, + a_end: usize, + b_byte: usize, + b_end: usize, +}; + +/// A hunk as the edit that applies it: replace `a[start..end]` with `replacement`. +pub const Edit = struct { + start: usize, + end: usize, + replacement: []const u8, +}; + +pub const Options = struct { + /// Lines that occur more often than this in a range are not anchors (git's default). + max_occurrences: u32 = 64, + /// Myers gives up on a range whose edit script is longer than this, and replaces the + /// range whole: the bound on a text where nearly every line changed. + max_cost: u32 = 1024, +}; + +pub const Error = error{TooLarge} || std.mem.Allocator.Error; + +const nil = std.math.maxInt(u32); +const Job = [4]u32; + +hunks: std.ArrayList(Hunk) = .empty, + +reps: std.ArrayList([]const u8) = .empty, +hashes: std.ArrayList(u64) = .empty, +table: std.ArrayList(u32) = .empty, +a_ids: std.ArrayList(u32) = .empty, +b_ids: std.ArrayList(u32) = .empty, +a_offs: std.ArrayList(usize) = .empty, +b_offs: std.ArrayList(usize) = .empty, +a_chg: std.ArrayList(u8) = .empty, +b_chg: std.ArrayList(u8) = .empty, +head: std.ArrayList(u32) = .empty, +next: std.ArrayList(u32) = .empty, +cnt: std.ArrayList(u32) = .empty, +stack: std.ArrayList(Job) = .empty, +v: std.ArrayList(i32) = .empty, +local: std.ArrayList([4]u32) = .empty, + +pub fn deinit(self: *Text, gpa: std.mem.Allocator) void { + self.hunks.deinit(gpa); + self.reps.deinit(gpa); + self.hashes.deinit(gpa); + self.table.deinit(gpa); + self.a_ids.deinit(gpa); + self.b_ids.deinit(gpa); + self.a_offs.deinit(gpa); + self.b_offs.deinit(gpa); + self.a_chg.deinit(gpa); + self.b_chg.deinit(gpa); + self.head.deinit(gpa); + self.next.deinit(gpa); + self.cnt.deinit(gpa); + self.stack.deinit(gpa); + self.v.deinit(gpa); + self.local.deinit(gpa); + self.* = .{}; +} + +/// `edit` for `h`, the replacement borrowed from `b`. +pub fn edit(h: Hunk, b: []const u8) Edit { + return .{ .start = h.a_byte, .end = h.a_end, .replacement = b[h.b_byte..h.b_end] }; +} + +/// A position in a document as a protocol names it (LSP: a line and a character in the +/// negotiated encoding). The library never knows the document type, so `range` takes the +/// owner's byte-to-position function. +pub const Position = struct { line: u32, character: u32 }; +pub const Range = struct { start: Position, end: Position }; + +/// `e`'s span in `a` as protocol positions, through `toPosition(ctx, byte) Position`. +pub fn range(e: Edit, ctx: anytype, comptime toPosition: fn (@TypeOf(ctx), usize) Position) Range { + return .{ .start = toPosition(ctx, e.start), .end = toPosition(ctx, e.end) }; +} + +/// The hunks that turn `a` into `b`, in order, valid until the next call. Equal texts give none. +pub fn diff(self: *Text, gpa: std.mem.Allocator, a: []const u8, b: []const u8, opts: Options) Error![]const Hunk { + self.hunks.clearRetainingCapacity(); + if (a.len >= nil or b.len >= nil) return error.TooLarge; + + // Common start and end, in whole lines. + var prefix = commonPrefix(a, b); + while (prefix > 0 and a[prefix - 1] != '\n') prefix -= 1; + const ra = a[prefix..]; + const rb = b[prefix..]; + if (ra.len == 0 and rb.len == 0) return self.hunks.items; + var s = commonSuffix(ra, rb); + // Back off to where both texts start a line. + while (s > 0) : (s -= 1) { + const p = ra.len - s; + const q = rb.len - s; + if ((p == 0 or ra[p - 1] == '\n') and (q == 0 or rb[q - 1] == '\n')) break; + } + const ma = ra[0 .. ra.len - s]; + const mb = rb[0 .. rb.len - s]; + const first_line: u32 = @intCast(std.mem.countScalar(u8, a[0..prefix], '\n')); + + // Lines of the middle, interned to ids. + const na_max = std.mem.countScalar(u8, ma, '\n') + 1; + const nb_max = std.mem.countScalar(u8, mb, '\n') + 1; + var size: usize = 16; + while (size < 2 * (na_max + nb_max)) size <<= 1; + try self.table.resize(gpa, size); + @memset(self.table.items, 0); + self.reps.clearRetainingCapacity(); + self.hashes.clearRetainingCapacity(); + try self.tokenize(gpa, ma, prefix, &self.a_offs, &self.a_ids); + try self.tokenize(gpa, mb, prefix, &self.b_offs, &self.b_ids); + const na: u32 = @intCast(self.a_ids.items.len); + const nb: u32 = @intCast(self.b_ids.items.len); + const unique: u32 = @intCast(self.reps.items.len); + + try self.a_chg.resize(gpa, na); + try self.b_chg.resize(gpa, nb); + @memset(self.a_chg.items, 0); + @memset(self.b_chg.items, 0); + try self.head.resize(gpa, unique); + try self.cnt.resize(gpa, unique); + @memset(self.head.items, nil); + @memset(self.cnt.items, 0); + try self.next.resize(gpa, na); + + try self.histogram(gpa, na, nb, opts); + try self.shift(gpa, na, nb); + + // The hunks, in absolute lines and bytes. + var i: u32 = 0; + var j: u32 = 0; + while (i < na or j < nb) { + if (i < na and j < nb and self.a_chg.items[i] == 0 and self.b_chg.items[j] == 0) { + i += 1; + j += 1; + continue; + } + const si = i; + const sj = j; + while (i < na and self.a_chg.items[i] != 0) i += 1; + while (j < nb and self.b_chg.items[j] != 0) j += 1; + std.debug.assert(i > si or j > sj); + try self.hunks.append(gpa, .{ + .a_start = first_line + si, + .a_len = i - si, + .b_start = first_line + sj, + .b_len = j - sj, + .a_byte = self.a_offs.items[si], + .a_end = self.a_offs.items[i], + .b_byte = self.b_offs.items[sj], + .b_end = self.b_offs.items[j], + }); + } + return self.hunks.items; +} + +/// Split `text` (which starts at byte `base` of its document) into lines, recording where each +/// starts (and, last, where the text ends) and its id. +fn tokenize(self: *Text, gpa: std.mem.Allocator, text: []const u8, base: usize, offs: *std.ArrayList(usize), ids: *std.ArrayList(u32)) Error!void { + offs.clearRetainingCapacity(); + ids.clearRetainingCapacity(); + const mask = self.table.items.len - 1; + var at: usize = 0; + while (at < text.len) { + const end = if (std.mem.indexOfScalarPos(u8, text, at, '\n')) |nl| nl + 1 else text.len; + const line = text[at..end]; + const h = std.hash.Wyhash.hash(0, line); + var slot: usize = @intCast(h & mask); + const id: u32 = while (true) : (slot = (slot + 1) & mask) { + const e = self.table.items[slot]; + if (e == 0) { + const new: u32 = @intCast(self.reps.items.len); + try self.reps.append(gpa, line); + try self.hashes.append(gpa, h); + self.table.items[slot] = new + 1; + break new; + } + if (self.hashes.items[e - 1] == h and std.mem.eql(u8, self.reps.items[e - 1], line)) break e - 1; + }; + try offs.append(gpa, base + at); + try ids.append(gpa, id); + at = end; + } + try offs.append(gpa, base + text.len); +} + +const lanes = 32; + +/// How many leading bytes agree. +fn commonPrefix(a: []const u8, b: []const u8) usize { + const n = @min(a.len, b.len); + var i: usize = 0; + while (i + lanes <= n) : (i += lanes) { + const x: @Vector(lanes, u8) = a[i..][0..lanes].*; + const y: @Vector(lanes, u8) = b[i..][0..lanes].*; + if (!@reduce(.And, x == y)) break; + } + while (i < n and a[i] == b[i]) i += 1; + return i; +} + +/// How many trailing bytes agree. +fn commonSuffix(a: []const u8, b: []const u8) usize { + const n = @min(a.len, b.len); + var i: usize = 0; + while (i + lanes <= n) : (i += lanes) { + const x: @Vector(lanes, u8) = a[a.len - i - lanes ..][0..lanes].*; + const y: @Vector(lanes, u8) = b[b.len - i - lanes ..][0..lanes].*; + if (!@reduce(.And, x == y)) break; + } + while (i < n and a[a.len - 1 - i] == b[b.len - 1 - i]) i += 1; + return i; +} + +/// Mark the changed lines of the whole of `a_ids` against `b_ids`. Ranges wait on an explicit +/// stack: a long text splits into thousands of them. +fn histogram(self: *Text, gpa: std.mem.Allocator, na: u32, nb: u32, opts: Options) Error!void { + self.stack.clearRetainingCapacity(); + try self.stack.append(gpa, .{ 0, na, 0, nb }); + const A = self.a_ids.items; + const B = self.b_ids.items; + while (self.stack.pop()) |r| { + var a_lo, var a_hi, var b_lo, var b_hi = r; + while (a_lo < a_hi and b_lo < b_hi and A[a_lo] == B[b_lo]) { + a_lo += 1; + b_lo += 1; + } + while (a_lo < a_hi and b_lo < b_hi and A[a_hi - 1] == B[b_hi - 1]) { + a_hi -= 1; + b_hi -= 1; + } + if (a_lo == a_hi or b_lo == b_hi) { + @memset(self.a_chg.items[a_lo..a_hi], 1); + @memset(self.b_chg.items[b_lo..b_hi], 1); + continue; + } + + // Where each id sits in A's range, ascending, and how often. + var i = a_hi; + while (i > a_lo) { + i -= 1; + const id = A[i]; + self.next.items[i] = self.head.items[id]; + self.head.items[id] = i; + self.cnt.items[id] += 1; + } + // The best match: the longest run, or the rarest lines. + var best: Job = .{ 0, 0, 0, 0 }; + var best_len: u32 = 0; + var best_rare: u32 = std.math.maxInt(u32); + var common = false; + var j = b_lo; + while (j < b_hi) { + const c = self.cnt.items[B[j]]; + var skip_to = j + 1; + if (c > 0) common = true; + if (c > 0 and c <= opts.max_occurrences) { + var p = self.head.items[B[j]]; + while (p != nil) : (p = self.next.items[p]) { + var sa = p; + var sb = j; + var ea = p + 1; + var eb = j + 1; + var rare = c; + while (sa > a_lo and sb > b_lo and A[sa - 1] == B[sb - 1]) { + sa -= 1; + sb -= 1; + rare = @min(rare, self.cnt.items[A[sa]]); + } + while (ea < a_hi and eb < b_hi and A[ea] == B[eb]) { + rare = @min(rare, self.cnt.items[A[ea]]); + ea += 1; + eb += 1; + } + if (ea - sa > best_len or (ea - sa == best_len and rare < best_rare)) { + best = .{ sa, ea, sb, eb }; + best_len = ea - sa; + best_rare = rare; + } + skip_to = @max(skip_to, eb); + } + } + j = skip_to; + } + i = a_lo; + while (i < a_hi) : (i += 1) { + self.head.items[A[i]] = nil; + self.cnt.items[A[i]] = 0; + } + + if (best_len > 0) { + try self.stack.append(gpa, .{ best[1], a_hi, best[3], b_hi }); + try self.stack.append(gpa, .{ a_lo, best[0], b_lo, best[2] }); + } else if (!common) { + @memset(self.a_chg.items[a_lo..a_hi], 1); + @memset(self.b_chg.items[b_lo..b_hi], 1); + } else { + try self.myers(gpa, a_lo, a_hi, b_lo, b_hi, opts.max_cost); + } + } +} + +/// Mark the changed lines of one range by Myers' O(ND) algorithm: find the middle snake from +/// both ends at once, then do each side of it. Past `max_cost` edits the range is replaced. +fn myers(self: *Text, gpa: std.mem.Allocator, a_lo0: u32, a_hi0: u32, b_lo0: u32, b_hi0: u32, max_cost: u32) Error!void { + var a_lo = a_lo0; + var a_hi = a_hi0; + var b_lo = b_lo0; + var b_hi = b_hi0; + const A = self.a_ids.items; + const B = self.b_ids.items; + while (a_lo < a_hi and b_lo < b_hi and A[a_lo] == B[b_lo]) { + a_lo += 1; + b_lo += 1; + } + while (a_lo < a_hi and b_lo < b_hi and A[a_hi - 1] == B[b_hi - 1]) { + a_hi -= 1; + b_hi -= 1; + } + if (a_lo == a_hi or b_lo == b_hi) { + @memset(self.a_chg.items[a_lo..a_hi], 1); + @memset(self.b_chg.items[b_lo..b_hi], 1); + return; + } + const a = A[a_lo..a_hi]; + const b = B[b_lo..b_hi]; + const n: i32 = @intCast(a.len); + const m: i32 = @intCast(b.len); + const max_d: i32 = @intCast(@min((a.len + b.len + 1) / 2, max_cost)); + const off = max_d + 1; + try self.v.resize(gpa, @intCast(4 * off + 4)); + const v1 = self.v.items[0..@intCast(2 * off + 2)]; + const v2 = self.v.items[@intCast(2 * off + 2)..][0..@intCast(2 * off + 2)]; + @memset(self.v.items, -1); + v1[@intCast(off + 1)] = 0; + v2[@intCast(off + 1)] = 0; + const delta = n - m; + const front = @mod(delta, 2) != 0; + var k1s: i32 = 0; + var k1e: i32 = 0; + var k2s: i32 = 0; + var k2e: i32 = 0; + var d: i32 = 0; + var split: ?[2]u32 = null; + search: while (d < max_d) : (d += 1) { + var k1 = -d + k1s; + while (k1 <= d - k1e) : (k1 += 2) { + const ko: usize = @intCast(off + k1); + var x1 = if (k1 == -d or (k1 != d and v1[ko - 1] < v1[ko + 1])) v1[ko + 1] else v1[ko - 1] + 1; + var y1 = x1 - k1; + while (x1 < n and y1 < m and a[@intCast(x1)] == b[@intCast(y1)]) { + x1 += 1; + y1 += 1; + } + v1[ko] = x1; + if (x1 > n) { + k1e += 2; + } else if (y1 > m) { + k1s += 2; + } else if (front) { + const k2o = off + delta - k1; + if (k2o >= 0 and k2o < v2.len and v2[@intCast(k2o)] != -1 and x1 >= n - v2[@intCast(k2o)]) { + split = .{ @intCast(x1), @intCast(y1) }; + break :search; + } + } + } + var k2 = -d + k2s; + while (k2 <= d - k2e) : (k2 += 2) { + const ko: usize = @intCast(off + k2); + var x2 = if (k2 == -d or (k2 != d and v2[ko - 1] < v2[ko + 1])) v2[ko + 1] else v2[ko - 1] + 1; + var y2 = x2 - k2; + while (x2 < n and y2 < m and a[@intCast(n - x2 - 1)] == b[@intCast(m - y2 - 1)]) { + x2 += 1; + y2 += 1; + } + v2[ko] = x2; + if (x2 > n) { + k2e += 2; + } else if (y2 > m) { + k2s += 2; + } else if (!front) { + const k1o = off + delta - k2; + if (k1o >= 0 and k1o < v1.len and v1[@intCast(k1o)] != -1) { + const x1 = v1[@intCast(k1o)]; + if (x1 >= n - x2) { + split = .{ @intCast(x1), @intCast(off + x1 - k1o) }; + break :search; + } + } + } + } + } + const sp = split orelse { + @memset(self.a_chg.items[a_lo..a_hi], 1); + @memset(self.b_chg.items[b_lo..b_hi], 1); + return; + }; + // Neither half can be the whole range again, since the snake is strictly inside it. + try self.myers(gpa, a_lo, a_lo + sp[0], b_lo, b_lo + sp[1], max_cost); + try self.myers(gpa, a_lo + sp[0], a_hi, b_lo + sp[1], b_hi, max_cost); +} + +/// Slide each pure insert or delete to where it reads best: a group of lines can move over an +/// unchanged line equal to its edge without changing what the diff means. Prefers ending on a +/// blank line, then before a less indented line; ties go lower. +fn shift(self: *Text, gpa: std.mem.Allocator, na: u32, nb: u32) Error!void { + // Which groups are pure, found before any slide moves them. + self.local.clearRetainingCapacity(); + var i: u32 = 0; + var j: u32 = 0; + while (i < na or j < nb) { + if (i < na and j < nb and self.a_chg.items[i] == 0 and self.b_chg.items[j] == 0) { + i += 1; + j += 1; + continue; + } + const si = i; + const sj = j; + while (i < na and self.a_chg.items[i] != 0) i += 1; + while (j < nb and self.b_chg.items[j] != 0) j += 1; + if ((i == si) != (j == sj)) try self.local.append(gpa, .{ si, i, sj, j }); + } + for (self.local.items) |g| { + if (g[0] != g[1]) self.slide(self.a_ids.items, self.a_chg.items, g[0], g[1]) else self.slide(self.b_ids.items, self.b_chg.items, g[2], g[3]); + } +} + +fn slide(self: *Text, ids: []const u32, chg: []u8, start: u32, end: u32) void { + var s = start; + var e = end; + // Another slide may have merged this group into a neighbour; leave those. + if ((s > 0 and chg[s - 1] != 0) or (e < chg.len and chg[e] != 0)) return; + for (chg[s..e]) |c| if (c == 0) return; + while (s > 0 and chg[s - 1] == 0 and ids[s - 1] == ids[e - 1]) { + chg[e - 1] = 0; + chg[s - 1] = 1; + s -= 1; + e -= 1; + } + var best_s = s; + var best = self.score(ids, e); + while (e < chg.len and chg[e] == 0 and ids[s] == ids[e]) { + chg[s] = 0; + chg[e] = 1; + s += 1; + e += 1; + const sc = self.score(ids, e); + if (sc >= best) { + best = sc; + best_s = s; + } + } + while (s > best_s) { + chg[e - 1] = 0; + chg[s - 1] = 1; + s -= 1; + e -= 1; + } +} + +/// How well a group ending before line `e` reads. +fn score(self: *Text, ids: []const u32, e: u32) i32 { + const last = self.reps.items[ids[e - 1]]; + var s: i32 = if (std.mem.trim(u8, last, " \t\r\n").len == 0) 1000 else 0; + if (e < ids.len) { + const after = self.reps.items[ids[e]]; + const indent = after.len - std.mem.trimStart(u8, after, " \t").len; + s -= @intCast(@min(indent, 200)); + } + return s; +} diff --git a/core/diff/tests.zig b/core/diff/tests.zig new file mode 100644 index 000000000..de8d694f6 --- /dev/null +++ b/core/diff/tests.zig @@ -0,0 +1,250 @@ +//! Tests for `core.diff` text: the hunks rebuild the new text from the old over random edits and +//! awkward bytes, hunks read well, a reused scratch allocates nothing, the unified output and +//! its cap. +const std = @import("std"); +const Text = @import("Text.zig"); +const unified = @import("unified.zig"); + +const gpa = std.testing.allocator; + +/// `a` with every hunk applied, as edits in order. +fn apply(out: *std.ArrayList(u8), a: []const u8, b: []const u8, hunks: []const Text.Hunk) !void { + var at: usize = 0; + for (hunks) |h| { + const e = Text.edit(h, b); + try out.appendSlice(gpa, a[at..e.start]); + try out.appendSlice(gpa, e.replacement); + at = e.end; + } + try out.appendSlice(gpa, a[at..]); +} + +fn expectRoundTrip(t: *Text, a: []const u8, b: []const u8) !void { + const hunks = try t.diff(gpa, a, b, .{}); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + try apply(&out, a, b, hunks); + try std.testing.expectEqualStrings(b, out.items); + // Hunks are ordered, apart and in step with the line counts. + var prev_a: u32 = 0; + for (hunks) |h| { + try std.testing.expect(h.a_len > 0 or h.b_len > 0); + try std.testing.expect(h.a_start >= prev_a); + prev_a = h.a_start + h.a_len; + try std.testing.expectEqual(h.a_len, @as(u32, @intCast(std.mem.countScalar(u8, a[h.a_byte..h.a_end], '\n') + @intFromBool(h.a_end > h.a_byte and a[h.a_end - 1] != '\n')))); + } +} + +test "equal texts have no hunks" { + var t: Text = .{}; + defer t.deinit(gpa); + try std.testing.expectEqual(0, (try t.diff(gpa, "", "", .{})).len); + try std.testing.expectEqual(0, (try t.diff(gpa, "a\nb\n", "a\nb\n", .{})).len); +} + +test "one changed line is one hunk" { + var t: Text = .{}; + defer t.deinit(gpa); + const hunks = try t.diff(gpa, "a\nb\nc\n", "a\nB\nc\n", .{}); + try std.testing.expectEqual(1, hunks.len); + try std.testing.expectEqual(1, hunks[0].a_start); + try std.testing.expectEqual(1, hunks[0].a_len); + try std.testing.expectEqual(1, hunks[0].b_len); +} + +test "two distant edits are two hunks" { + var t: Text = .{}; + defer t.deinit(gpa); + const a = "1\n2\n3\n4\n5\n6\n7\n8\n"; + const b = "1\nx\n3\n4\n5\n6\ny\n8\n"; + try std.testing.expectEqual(2, (try t.diff(gpa, a, b, .{})).len); + try expectRoundTrip(&t, a, b); +} + +test "inserts and deletes at the ends and in empty texts" { + var t: Text = .{}; + defer t.deinit(gpa); + try expectRoundTrip(&t, "", "a\n"); + try expectRoundTrip(&t, "a\n", ""); + try expectRoundTrip(&t, "b\n", "a\nb\nc\n"); + try expectRoundTrip(&t, "a\nb\nc\n", "b\n"); + try expectRoundTrip(&t, "a", "a\n"); + try expectRoundTrip(&t, "a\nb", "a\nb\r\n"); +} + +test "a block moved up reads as one insert, not a rewrite" { + var t: Text = .{}; + defer t.deinit(gpa); + const a = "fn a() {\n}\n\nfn c() {\n}\n"; + const b = "fn a() {\n}\n\nfn b() {\n}\n\nfn c() {\n}\n"; + const hunks = try t.diff(gpa, a, b, .{}); + try std.testing.expectEqual(1, hunks.len); + try std.testing.expectEqual(0, hunks[0].a_len); + try std.testing.expectEqual(3, hunks[0].b_len); + // Slid to end on the blank line, not start on one. + try std.testing.expectEqualStrings("fn b() {\n}\n\n", b[hunks[0].b_byte..hunks[0].b_end]); +} + +test "histogram's anchors: common braces do not pull the diff off the real lines" { + var t: Text = .{}; + defer t.deinit(gpa); + const a = "int f() {\n return 1;\n}\n\nint g() {\n return 2;\n}\n"; + const b = "int g() {\n return 2;\n}\n\nint f() {\n return 1;\n}\n"; + try expectRoundTrip(&t, a, b); +} + +test "the cost cap replaces the range, and the result still applies" { + var t: Text = .{}; + defer t.deinit(gpa); + var a: std.ArrayList(u8) = .empty; + defer a.deinit(gpa); + var b: std.ArrayList(u8) = .empty; + defer b.deinit(gpa); + // Only two distinct lines, so histogram has no anchor under the cap and Myers runs. + var prng: std.Random.DefaultPrng = .init(5); + const r = prng.random(); + for (0..400) |_| { + try a.appendSlice(gpa, if (r.boolean()) "x\n" else "y\n"); + try b.appendSlice(gpa, if (r.boolean()) "x\n" else "y\n"); + } + for ([_]u32{ 1024, 4 }) |cost| { + const hunks = try t.diff(gpa, a.items, b.items, .{ .max_occurrences = 2, .max_cost = cost }); + var out: std.ArrayList(u8) = .empty; + defer out.deinit(gpa); + try apply(&out, a.items, b.items, hunks); + try std.testing.expectEqualStrings(b.items, out.items); + } +} + +fn randomText(r: std.Random, w: *std.ArrayList(u8), lines: usize, alphabet: []const u8) !void { + for (0..lines) |_| { + for (0..r.uintLessThan(usize, 5)) |_| try w.append(gpa, alphabet[r.uintLessThan(usize, alphabet.len)]); + if (r.uintLessThan(u8, 12) != 0) try w.append(gpa, '\n'); + } +} + +test "random edits round trip, over text, CRLF, no final newline and binary" { + var t: Text = .{}; + defer t.deinit(gpa); + var prng: std.Random.DefaultPrng = .init(0xd1ff); + const r = prng.random(); + const alphabets = [_][]const u8{ "ab\n", "ab\r\n", "a\n\n\n", "\x00\xff\n\r ab" }; + for (0..300) |n| { + var a: std.ArrayList(u8) = .empty; + defer a.deinit(gpa); + try randomText(r, &a, r.uintLessThan(usize, 60), alphabets[n % alphabets.len]); + // b is a with a few random splices. + var b: std.ArrayList(u8) = .empty; + defer b.deinit(gpa); + try b.appendSlice(gpa, a.items); + for (0..r.uintLessThan(usize, 5)) |_| { + const at = if (b.items.len == 0) 0 else r.uintLessThan(usize, b.items.len); + const cut = @min(r.uintLessThan(usize, 20), b.items.len - at); + var piece: std.ArrayList(u8) = .empty; + defer piece.deinit(gpa); + try randomText(r, &piece, r.uintLessThan(usize, 4), alphabets[n % alphabets.len]); + try b.replaceRange(gpa, at, cut, piece.items); + } + try expectRoundTrip(&t, a.items, b.items); + try expectRoundTrip(&t, b.items, a.items); + } +} + +test "non-overlapping edits keep their hunks apart" { + var t: Text = .{}; + defer t.deinit(gpa); + var a: std.ArrayList(u8) = .empty; + defer a.deinit(gpa); + for (0..200) |i| try a.print(gpa, "line {d}\n", .{i}); + var b: std.ArrayList(u8) = .empty; + defer b.deinit(gpa); + var it = std.mem.splitScalar(u8, a.items, '\n'); + var n: usize = 0; + while (it.next()) |l| : (n += 1) { + if (l.len == 0) continue; + if (n % 50 == 7) try b.print(gpa, "changed {d}\n", .{n}) else try b.print(gpa, "{s}\n", .{l}); + } + try std.testing.expectEqual(4, (try t.diff(gpa, a.items, b.items, .{})).len); +} + +test "a reused scratch allocates nothing" { + var t: Text = .{}; + defer t.deinit(gpa); + const a = "a\nb\nc\nd\ne\nf\ng\n"; + const b = "a\nB\nc\nd\nE\nf\ng\nh\n"; + _ = try t.diff(gpa, a, b, .{}); + var failing = std.testing.FailingAllocator.init(gpa, .{ .fail_index = 0 }); + const hunks = try t.diff(failing.allocator(), a, b, .{}); + try std.testing.expectEqual(3, hunks.len); +} + +fn render(a: []const u8, b: []const u8, opts: unified.Options) !struct { text: []u8, omitted: u32 } { + var t: Text = .{}; + defer t.deinit(gpa); + const hunks = try t.diff(gpa, a, b, .{}); + var out: std.Io.Writer.Allocating = .init(gpa); + errdefer out.deinit(); + const omitted = try unified.write(gpa, &out.writer, "a/f", "b/f", a, b, hunks, opts); + return .{ .text = try out.toOwnedSlice(), .omitted = omitted }; +} + +test "unified output" { + const a = "1\n2\n3\n4\n5\n6\n7\n8\n9\n10\n11\n12\n"; + const b = "1\n2\n3\nfour\n5\n6\n7\n8\n9\n10\n11\n12\n13\n"; + const got = try render(a, b, .{ .context = 2 }); + defer gpa.free(got.text); + try std.testing.expectEqualStrings( + \\--- a/f + \\+++ b/f + \\@@ -2,5 +2,5 @@ + \\ 2 + \\ 3 + \\-4 + \\+four + \\ 5 + \\ 6 + \\@@ -11,2 +11,3 @@ + \\ 11 + \\ 12 + \\+13 + \\ + , got.text); +} + +test "unified marks a missing final newline and writes pure inserts and deletes" { + const got = try render("a\nb", "a\nb\nc\n", .{ .context = 1 }); + defer gpa.free(got.text); + try std.testing.expectEqualStrings( + \\--- a/f + \\+++ b/f + \\@@ -1,2 +1,3 @@ + \\ a + \\-b + \\\ No newline at end of file + \\+b + \\+c + \\ + , got.text); + const gone = try render("a\n", "", .{}); + defer gpa.free(gone.text); + try std.testing.expectEqualStrings("--- a/f\n+++ b/f\n@@ -1 +0,0 @@\n-a\n", gone.text); +} + +test "unified cap leaves whole groups and counts the rest" { + var a: std.ArrayList(u8) = .empty; + defer a.deinit(gpa); + var b: std.ArrayList(u8) = .empty; + defer b.deinit(gpa); + for (0..100) |i| { + try a.print(gpa, "line {d}\n", .{i}); + if (i % 20 == 5) try b.print(gpa, "edit {d}\n", .{i}) else try b.print(gpa, "line {d}\n", .{i}); + } + const full = try render(a.items, b.items, .{}); + defer gpa.free(full.text); + try std.testing.expectEqual(0, full.omitted); + const capped = try render(a.items, b.items, .{ .max_bytes = 250 }); + defer gpa.free(capped.text); + try std.testing.expect(capped.omitted > 0 and capped.omitted < 5); + try std.testing.expect(capped.text.len <= 250); + try std.testing.expect(std.mem.indexOf(u8, capped.text, " more hunk") != null); +} diff --git a/core/diff/unified.zig b/core/diff/unified.zig new file mode 100644 index 000000000..39e05178e --- /dev/null +++ b/core/diff/unified.zig @@ -0,0 +1,128 @@ +//! Standard unified diff (`@@ -a,l +b,l @@`) of two texts from their hunks: what `patch` and +//! `git apply` accept and what an agent reads best, sized by a byte cap. +const std = @import("std"); +const Text = @import("Text.zig"); + +pub const Options = struct { + /// Unchanged lines kept around each change. + context: u32 = 3, + /// Stop before the hunk group that would pass this many bytes (0: no limit). The output ends + /// with a line saying how many hunks were left out; a group is never cut in half, so the + /// output stays a valid patch, and the first group is dropped too if it alone is over. + max_bytes: usize = 0, +}; + +/// Write `hunks` (from `Text.diff(a, b)`) as a unified diff headed by the two names. Returns +/// how many hunks the cap left out. +pub fn write(gpa: std.mem.Allocator, w: *std.Io.Writer, a_name: []const u8, b_name: []const u8, a: []const u8, b: []const u8, hunks: []const Text.Hunk, opts: Options) (std.Io.Writer.Error || std.mem.Allocator.Error)!u32 { + if (hunks.len == 0) return 0; + var head: std.Io.Writer.Allocating = .init(gpa); + defer head.deinit(); + try head.writer.print("--- {s}\n+++ {s}\n", .{ a_name, b_name }); + var written = head.written().len; + try w.writeAll(head.written()); + + var group: std.Io.Writer.Allocating = .init(gpa); + defer group.deinit(); + var i: usize = 0; + while (i < hunks.len) { + // Hunks whose context would touch belong to one `@@` group. + var end = i + 1; + while (end < hunks.len and gapLines(hunks[end - 1], hunks[end]) <= 2 * opts.context) end += 1; + group.clearRetainingCapacity(); + try writeGroup(&group.writer, a, b, hunks[i..end], opts.context); + if (opts.max_bytes != 0 and written + group.written().len + trailer_room > opts.max_bytes) { + const left: u32 = @intCast(hunks.len - i); + try w.print("... {d} more hunk{s}\n", .{ left, if (left == 1) "" else "s" }); + return left; + } + try w.writeAll(group.written()); + written += group.written().len; + i = end; + } + return 0; +} + +/// Bytes kept free for the "... N more hunks" line. +const trailer_room = 32; + +fn gapLines(prev: Text.Hunk, next: Text.Hunk) u32 { + return next.a_start - (prev.a_start + prev.a_len); +} + +/// Where the line `n` lines before byte `at` (a line start) begins, not before the text's start. +fn backLines(text: []const u8, at: usize, n: u32) usize { + var pos = at; + var left = n; + while (left > 0 and pos > 0) : (left -= 1) { + pos = if (std.mem.lastIndexOfScalar(u8, text[0 .. pos - 1], '\n')) |nl| nl + 1 else 0; + } + return pos; +} + +/// Where the text `n` lines past byte `at` ends (the end of text at most). +fn forwardLines(text: []const u8, at: usize, n: u32) usize { + var pos = at; + var left = n; + while (left > 0 and pos < text.len) : (left -= 1) { + pos = if (std.mem.indexOfScalarPos(u8, text, pos, '\n')) |nl| nl + 1 else text.len; + } + return pos; +} + +fn lineCount(text: []const u8) u32 { + return @intCast(std.mem.countScalar(u8, text, '\n') + @intFromBool(text.len > 0 and text[text.len - 1] != '\n')); +} + +fn writeGroup(w: *std.Io.Writer, a: []const u8, b: []const u8, hunks: []const Text.Hunk, context: u32) !void { + const first = hunks[0]; + const last = hunks[hunks.len - 1]; + const lead = backLines(a, first.a_byte, context); + const lead_lines = lineCount(a[lead..first.a_byte]); + const trail = forwardLines(a, last.a_end, context); + var a_n: u32 = lead_lines + lineCount(a[last.a_end..trail]); + var b_n: u32 = a_n; + for (hunks, 0..) |h, k| { + a_n += h.a_len; + b_n += h.b_len; + // Between hunks the context counts on both sides. + if (k > 0) { + const gap = gapLines(hunks[k - 1], h); + a_n += gap; + b_n += gap; + } + } + try w.writeAll("@@ -"); + try span(w, first.a_start - lead_lines, a_n); + try w.writeAll(" +"); + try span(w, first.b_start - lead_lines, b_n); + try w.writeAll(" @@\n"); + + var at = lead; + for (hunks) |h| { + try lines(w, ' ', a[at..h.a_byte]); + try lines(w, '-', a[h.a_byte..h.a_end]); + try lines(w, '+', b[h.b_byte..h.b_end]); + at = h.a_end; + } + try lines(w, ' ', a[at..trail]); +} + +/// `start,len` as unified diff writes it: 1-based, the line before for an empty range, and +/// no length when it is one. +fn span(w: *std.Io.Writer, from: u32, len: u32) !void { + if (len == 0) return w.print("{d},0", .{from}); + if (len == 1) return w.print("{d}", .{from + 1}); + try w.print("{d},{d}", .{ from + 1, len }); +} + +fn lines(w: *std.Io.Writer, mark: u8, text: []const u8) !void { + var at: usize = 0; + while (at < text.len) { + const end = if (std.mem.indexOfScalarPos(u8, text, at, '\n')) |nl| nl + 1 else text.len; + try w.writeByte(mark); + try w.writeAll(text[at..end]); + if (text[end - 1] != '\n') try w.writeAll("\n\\ No newline at end of file\n"); + at = end; + } +} diff --git a/tests/bench/bench_diff.zig b/tests/bench/bench_diff.zig index 7c5acedb8..e53717861 100644 --- a/tests/bench/bench_diff.zig +++ b/tests/bench/bench_diff.zig @@ -128,3 +128,91 @@ test "bench: 50-layer 2048x2048 sprite, one layer touched" { std.debug.print("\n== core.diff, {d} layers of {d}x{d} ==\none layer touched {d:8.3} ms (target < 10)\n", .{ layers, n, n, ms(shared_ns) }); std.debug.print(" one layer, distinct copy {d:8.3} ms\n", .{ms(try bestOf(5, p, runDiff))}); } + +// ---- text (step 1) ---- + +const tgpa = std.heap.page_allocator; +const text_lines = 25_000; + +/// Source-shaped lines: indentation, short common ones (braces, blanks) and unique ones. +fn source(out: *std.ArrayList(u8), r: std.Random) !void { + for (0..text_lines) |i| { + switch (r.uintLessThan(u8, 8)) { + 0 => try out.appendSlice(tgpa, "}\n"), + 1 => try out.appendSlice(tgpa, "\n"), + else => try out.print(tgpa, "{s}const value_{d} = compute(arg_{d}, {d}); // line {d}\n", .{ " "[0 .. 4 * r.uintLessThan(usize, 3)], i, i % 97, r.int(u16), i }), + } + } +} + +/// `a` with every `every`-th line replaced (0: none). +fn edited(out: *std.ArrayList(u8), a: []const u8, every: usize, tag: []const u8) !void { + var it = std.mem.splitScalar(u8, a, '\n'); + var n: usize = 0; + while (it.next()) |l| : (n += 1) { + if (it.peek() == null and l.len == 0) break; + if (every != 0 and n % every == 0) try out.print(tgpa, " {s} {d}\n", .{ tag, n }) else try out.print(tgpa, "{s}\n", .{l}); + } +} + +fn check(a: []const u8, b: []const u8, hunks: []const diff.Text.Hunk) !void { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(tgpa); + var at: usize = 0; + for (hunks) |h| { + const e = diff.Text.edit(h, b); + try out.appendSlice(tgpa, a[at..e.start]); + try out.appendSlice(tgpa, e.replacement); + at = e.end; + } + try out.appendSlice(tgpa, a[at..]); + try std.testing.expectEqualStrings(b, out.items); +} + +fn run(t: *diff.Text, name: []const u8, budget_ms: f64, a: []const u8, b: []const u8) !void { + var hunks: []const diff.Text.Hunk = &.{}; + _ = try t.diff(tgpa, a, b, .{}); // warm the scratch + var best: i96 = std.math.maxInt(i96); + for (0..10) |_| { + const t0 = nowNs(); + hunks = try t.diff(tgpa, a, b, .{}); + best = @min(best, nowNs() - t0); + } + try check(a, b, hunks); + const took = @as(f64, @floatFromInt(best)) / 1e6; + std.debug.print(" {s:<34} {d:>9.3} ms (budget {d:>5.1} ms) {d} hunks {s}\n", .{ name, took, budget_ms, hunks.len, if (took <= budget_ms) "ok" else "OVER" }); +} + +test "bench: text, 1 MB" { + var prng: std.Random.DefaultPrng = .init(0xbe7c); + var a: std.ArrayList(u8) = .empty; + try source(&a, prng.random()); + std.debug.print("\nbench-diff: {d} bytes, {d} lines\n", .{ a.items.len, text_lines }); + var t: diff.Text = .{}; + + try run(&t, "identical", 0.2, a.items, a.items); + + var one: std.ArrayList(u8) = .empty; + try one.appendSlice(tgpa, a.items); + const mid = std.mem.indexOfScalarPos(u8, one.items, a.items.len / 2, '\n').? + 1; + try one.insertSlice(tgpa, mid, " one edited line\n"); + try run(&t, "one line inserted", 2, a.items, one.items); + + var scattered: std.ArrayList(u8) = .empty; + try edited(&scattered, a.items, 100, "edited"); + try run(&t, "1% of lines edited", 15, a.items, scattered.items); + + var all: std.ArrayList(u8) = .empty; + try edited(&all, a.items, 1, "rewritten"); + try run(&t, "every line changed", 60, a.items, all.items); + + // Lines that repeat past the occurrence cap, so Myers does the work. + var rep_a: std.ArrayList(u8) = .empty; + var rep_b: std.ArrayList(u8) = .empty; + var r = prng.random(); + for (0..text_lines) |_| { + try rep_a.appendSlice(tgpa, if (r.boolean()) "x\n" else "y\n"); + try rep_b.appendSlice(tgpa, if (r.boolean()) "x\n" else "y\n"); + } + try run(&t, "two-line alphabet (Myers, capped)", 60, rep_a.items, rep_b.items); +}