From 0478ea24570b586cbd8b94ef7408a2a14f67b32d Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 2 Oct 2026 22:23:15 +0000 Subject: [PATCH] feat(go): plantest.Golden checks in the contexts a policy produces A policy derives each field's encryption context from names, so an ordinary rename (a proto field, a Go struct field, a table) can change a context with no error on the write path: rows already written stop decrypting and their query terms stop matching. plantest.Golden makes that a test failure. Golden builds the plan as plan.PlanFor does at startup and compares what it stores each field as with a snapshot checked in at testdata/.golden; -update rewrites it. The snapshot records only what stored rows depend on: per encrypted field its column, context, index terms and facts, keyed by column rather than field name, so a rename the policy pins with plan.Column leaves it byte-for-byte unchanged and the test passes; per field decided Plaintext, its name and facts. Output is sorted and line-ending-normalised, so it is the same on every run and platform. On a mismatch the failure sorts the changes by cost: context changes (data loss) first, target changes (a migration) next, then the rest, followed by a unified diff. For a lost context it names the pin that keeps it, and only after building the plan with that pin to confirm it restores the old column and context. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01U5Y8iJ3ZM71digszHzpyrd --- languages/golang/stackencrypt/README.md | 45 ++ languages/golang/stackencrypt/plan/doc.go | 10 + .../stackencrypt/plan/plantest/compare.go | 385 ++++++++++++++ .../stackencrypt/plan/plantest/golden_test.go | 47 ++ .../stackencrypt/plan/plantest/plantest.go | 185 +++++++ .../plan/plantest/plantest_internal_test.go | 471 ++++++++++++++++++ .../stackencrypt/plan/plantest/snapshot.go | 282 +++++++++++ .../testdata/TestPolicies/audits.golden | 8 + .../testdata/TestPolicies/individuals.golden | 28 ++ 9 files changed, 1461 insertions(+) create mode 100644 languages/golang/stackencrypt/plan/plantest/compare.go create mode 100644 languages/golang/stackencrypt/plan/plantest/golden_test.go create mode 100644 languages/golang/stackencrypt/plan/plantest/plantest.go create mode 100644 languages/golang/stackencrypt/plan/plantest/plantest_internal_test.go create mode 100644 languages/golang/stackencrypt/plan/plantest/snapshot.go create mode 100644 languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/audits.golden create mode 100644 languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/individuals.golden diff --git a/languages/golang/stackencrypt/README.md b/languages/golang/stackencrypt/README.md index bef131e87..824ec32fe 100644 --- a/languages/golang/stackencrypt/README.md +++ b/languages/golang/stackencrypt/README.md @@ -296,6 +296,51 @@ their record key, and `plan.Identity` is refused. The plan a policy builds is a `Plan` like any other: the guest receives the same bytes as for the equivalent hand-built plan. +### Checking contexts in with a golden test + +Nothing on the write path notices a changed context: rename a proto or +struct field with no pin and new rows are simply written under a new one, +while the rows already written stop decrypting. The `plan/plantest` package +turns that into a test failure: + +```go +import "github.com/cipherstash/stack/languages/golang/stackencrypt/plan/plantest" + +func TestIndividualsPolicy(t *testing.T) { + plantest.Golden(t, source, Individuals) +} +``` + +Run it once with `go test -run '^TestIndividualsPolicy$' -update` to write +`testdata/TestIndividualsPolicy.golden`, and check the file in. It lists the +message's table and, for every field the policy decides, what it is stored +as: an encrypted field's column, context, index terms and facts, or a +plaintext field's name and facts. + +```text +table individuals + +column email + context individuals/email + terms eq match + fact fides.data_categories user.contact.email + +column medicare_number + context individuals/medicare_number + terms eq + fact fides.data_categories user.government_id +``` + +From then on the test builds the plan as `MustPlanFor` does at startup and +fails when it no longer matches the file, sorting the changes by what they +cost. A changed context is data loss and is reported first, with the +`plan.Column` (or `plan.Identity`) pin that keeps it; a changed target, such +as different index terms or a plaintext field now encrypted, is a migration; +anything else, such as a new field, is reported last. Encrypted fields are +listed by column, not by field name, so a rename the policy pins leaves the +file unchanged and the test passes. When a change is intended, rerun with +`-update` and review the diff. + ## Errors Errors are sentinel values, matched with `errors.Is`. The wasm guest diff --git a/languages/golang/stackencrypt/plan/doc.go b/languages/golang/stackencrypt/plan/doc.go index 385da3a99..3614c8457 100644 --- a/languages/golang/stackencrypt/plan/doc.go +++ b/languages/golang/stackencrypt/plan/doc.go @@ -67,6 +67,16 @@ // A [Custom] target supplies its context itself; [Column] names only its // record key, and [Identity] is refused. // +// Nothing on the write path notices a context that changed: a rename with +// no pin simply writes new rows under a new context. Check them in with a +// golden test ([github.com/cipherstash/stack/languages/golang/stackencrypt/plan/plantest.Golden]), +// which snapshots what the policy stores each field as and fails, naming +// the pin, when a context changes: +// +// func TestIndividualsPolicy(t *testing.T) { +// plantest.Golden(t, source, Individuals) +// } +// // # Failing closed // // A field with annotations that no rule decides is an error when the plan diff --git a/languages/golang/stackencrypt/plan/plantest/compare.go b/languages/golang/stackencrypt/plan/plantest/compare.go new file mode 100644 index 000000000..baad1c8cb --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/compare.go @@ -0,0 +1,385 @@ +package plantest + +import ( + "fmt" + "path/filepath" + "slices" + "strings" + + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" +) + +// changes is what differs between a snapshot and the policy now, sorted +// by what each costs. +type changes struct { + // contexts lose data: a context no field writes any more. + contexts []string + // migrations need rows rewritten: a target that changed under a + // context that did not. + migrations []string + // other changes cost nothing already written: a new field, a changed + // fact, a field the policy no longer decides. + other []string +} + +// explain says what changed between the snapshot want and the policy now. +func explain(want []byte, cur snapshot, facts []plan.Fact, m plan.Message) string { + old, err := parse(want) + if err != nil { + return fmt.Sprintf("The snapshot does not parse (%v), so its changes cannot be sorted; the diff below shows them.\n\n", err) + } + return compare(old, cur, facts, m).String() +} + +func (c changes) String() string { + var b strings.Builder + section := func(title string, items []string) { + if len(items) == 0 { + return + } + b.WriteString(title) + b.WriteString("\n") + for _, it := range items { + fmt.Fprintf(&b, " - %s\n", it) + } + b.WriteString("\n") + } + section("CONTEXT CHANGES. These lose data: a context is bound into every ciphertext and query term written under it, so rows already written under the old one no longer decrypt, and their terms no longer match queries. Keep the old context by pinning the field's rule with plan.Column (and plan.Identity after a database column rename).", c.contexts) + section("TARGET CHANGES. These need a migration: rows already written keep what they were written with until they are rewritten.", c.migrations) + section("OTHER CHANGES.", c.other) + if b.Len() == 0 { + return "Nothing the policy stores changed, but the snapshot's text did: an edit by hand, or a newer plantest.\n\n" + } + return b.String() +} + +// compare sorts the differences between the snapshot old and the policy +// now (cur, built from facts by m). +func compare(old, cur snapshot, facts []plan.Fact, m plan.Message) changes { + var c changes + if old.table != cur.table { + c.contexts = append(c.contexts, fmt.Sprintf("the message's table is %s, was %s. The table is the first half of every EQL column's context; restore plan.Table(%q).", + token(cur.table), token(old.table), old.table)) + } + + oldCols, curCols := byName(old.columns), byName(cur.columns) + oldPlain, curPlain := byField(old.plaintext), byField(cur.plaintext) + oldContexts := map[string]bool{} + for _, o := range old.columns { + oldContexts[o.context] = true + } + // The entries now that an old one accounts for, and the old plaintext + // fields an entry now accounts for. + seenCol, seenPlain, seenOldPlain := map[string]bool{}, map[string]bool{}, map[string]bool{} + + for _, o := range old.columns { + if n, ok := curCols[o.name]; ok { + seenCol[n.name] = true + if n.context != o.context { + c.contexts = append(c.contexts, fmt.Sprintf("column %s: its context is %q, was %q. %s", + token(o.name), n.context, o.context, pinAdvice(m, facts, n.from, o, old.table))) + } + c.stored(o, n) + continue + } + // A database column rename: a new column under the same context. + if n, ok := only(cur.columns, func(n column) bool { + _, before := oldCols[n.name] + return !before && !seenCol[n.name] && n.context == o.context + }); ok { + seenCol[n.name] = true + c.migrations = append(c.migrations, fmt.Sprintf("column %s is now stored in column %s, under the same context: the database column is renamed with it (ALTER TABLE ... RENAME COLUMN).", + token(o.name), token(n.name))) + c.stored(o, n) + continue + } + // Decided Plaintext now: under the column's name, or the only new + // plaintext field with its facts. + if p, ok := only(cur.plaintext, func(p plain) bool { + _, before := oldPlain[p.field] + return !before && !seenPlain[p.field] && (p.field == o.name || len(o.facts) > 0 && slices.Equal(p.facts, o.facts)) + }); ok { + seenPlain[p.field] = true + c.migrations = append(c.migrations, fmt.Sprintf("column %s is now field %s, decided Plaintext. Rows already written hold ciphertexts under %q, which a migration must decrypt before the field is read as plaintext.", + token(o.name), token(p.field), o.context)) + continue + } + // The context is lost. The only new column with the same facts, + // under a context new to the snapshot, is most likely the same + // field renamed. + msg := fmt.Sprintf("column %s: no field writes its context %q any more.", token(o.name), o.context) + if n, ok := only(cur.columns, func(n column) bool { + _, before := oldCols[n.name] + return !before && !seenCol[n.name] && !oldContexts[n.context] && len(o.facts) > 0 && slices.Equal(n.facts, o.facts) + }); ok { + seenCol[n.name] = true + msg += fmt.Sprintf(" Field %s now writes column %s under %q with the same facts, so it is likely the same field renamed. %s", + fieldName(n.from), token(n.name), n.context, pinAdvice(m, facts, n.from, o, old.table)) + c.stored(o, n) + } else { + msg += fmt.Sprintf(" If its field was renamed, pin the renamed field's rule with plan.Column(%q); if it was removed, rows written under it can no longer be read through this policy.", o.name) + } + c.contexts = append(c.contexts, msg) + } + + for _, n := range cur.columns { + if seenCol[n.name] { + continue + } + if n.from != nil { + if p, ok := oldPlain[n.from.fact.Field]; ok { + if _, still := curPlain[p.field]; !still { + seenOldPlain[p.field] = true + c.migrations = append(c.migrations, fmt.Sprintf("field %s, decided Plaintext before, is now encrypted into column %s under %q. Rows already written hold its plaintext, which a migration must encrypt.", + token(p.field), token(n.name), n.context)) + continue + } + } + } + c.other = append(c.other, fmt.Sprintf("new column %s, under %q, with terms [%s].", token(n.name), n.context, termList(n.terms))) + } + + for _, o := range old.plaintext { + if n, ok := curPlain[o.field]; ok { + if !slices.Equal(o.facts, n.facts) { + c.other = append(c.other, fmt.Sprintf("plaintext field %s: its facts are [%s], were [%s].", token(o.field), factList(n.facts), factList(o.facts))) + } + continue + } + if !seenOldPlain[o.field] { + c.other = append(c.other, fmt.Sprintf("plaintext field %s is no longer decided by the policy (renamed, removed, or no longer classified).", token(o.field))) + } + } + for _, n := range cur.plaintext { + if _, before := oldPlain[n.field]; !before && !seenPlain[n.field] { + c.other = append(c.other, fmt.Sprintf("new plaintext field %s.", token(n.field))) + } + } + return c +} + +// stored compares what one column stores, the context aside. +func (c *changes) stored(o, n column) { + if !slices.Equal(o.terms, n.terms) { + c.migrations = append(c.migrations, fmt.Sprintf("column %s: its terms are [%s], were [%s]. Rows already written carry the terms they were written with until they are re-encrypted, so a query on a new term misses them.", + token(n.name), termList(n.terms), termList(o.terms))) + } + if !slices.Equal(o.facts, n.facts) { + c.other = append(c.other, fmt.Sprintf("column %s: its facts are [%s], were [%s].", token(n.name), factList(n.facts), factList(o.facts))) + } +} + +// pinAdvice says how to store the field from in the old column under the +// old context again: the old table, when the context was under it and the +// table moved; the pin that does, checked by building the plan with it; or +// why no pin can. +func pinAdvice(m plan.Message, facts []plan.Fact, from *decided, old column, oldTable string) string { + if from == nil { + return "" + } + if oldTable != string(m.Table()) && strings.HasPrefix(old.context, oldTable+"/") { + return fmt.Sprintf("Restoring plan.Table(%q) brings it back.", oldTable) + } + if pin := pin(m, facts, from, old.name, old.context); pin != "" { + return fmt.Sprintf("Pinning the rule that decides field %s with %s keeps it.", fieldName(from), pin) + } + return fmt.Sprintf("No plan.Column or plan.Identity pin on the rule that decides field %s brings it back: the context comes from the target itself (a plan.Custom context, or a change of target), so restore that.", fieldName(from)) +} + +// pin is the rule options, spelled as Go, that store the field from as +// column under context again, checked by building the plan with them; "" +// when none does. +func pin(m plan.Message, facts []plan.Fact, from *decided, column, context string) string { + if column == "" { + return "" + } + type try struct { + spelled string + opts []plan.RuleOption + } + tries := []try{{fmt.Sprintf("plan.Column(%q)", column), []plan.RuleOption{plan.Column(column)}}} + if id, ok := strings.CutPrefix(context, string(m.Table())+"/"); ok && id != "" { + tries = append(tries, + try{fmt.Sprintf("plan.Identity(%q)", id), []plan.RuleOption{plan.Identity(id)}}, + try{fmt.Sprintf("plan.Column(%q), plan.Identity(%q)", column, id), []plan.RuleOption{plan.Column(column), plan.Identity(id)}}, + ) + } + for _, t := range tries { + rule := plan.When(plan.Field(from.fact.Field), from.decision, t.opts...) + p, err := plan.ForMessage(m.Msg(), m.Table(), rule.OrElse(m.Decide)).Build(facts) + if err != nil { + continue + } + for _, fp := range p.Fields() { + if fp.Field == goField(from.fact) && fp.Name == column && fp.Context == context { + return t.spelled + } + } + } + return "" +} + +// only is the one element of s that keep accepts, if exactly one does. +func only[T any](s []T, keep func(T) bool) (T, bool) { + var found T + n := 0 + for _, v := range s { + if keep(v) { + found = v + n++ + } + } + if n != 1 { + var zero T + return zero, false + } + return found, true +} + +func byName(cs []column) map[string]column { + out := make(map[string]column, len(cs)) + for _, c := range cs { + out[c.name] = c + } + return out +} + +func byField(ps []plain) map[string]plain { + out := make(map[string]plain, len(ps)) + for _, p := range ps { + out[p.field] = p + } + return out +} + +// fieldName names a field the way a rule matches it, and the Go field it +// binds to when that is spelled differently. +func fieldName(d *decided) string { + if d == nil { + return "?" + } + name := token(d.fact.Field) + if g := goField(d.fact); g != d.fact.Field { + name += " (" + g + ")" + } + return name +} + +func factList(fs []fact) string { + if len(fs) == 0 { + return "none" + } + parts := make([]string, len(fs)) + for i, f := range fs { + parts[i] = token(f.key) + "=" + token(f.value) + } + return strings.Join(parts, " ") +} + +// diff is a unified diff of the snapshot want and the policy now, with +// two lines of context. +func diff(path string, want, got []byte) string { + es := edits(lines(want), lines(got)) + const context = 2 + keep := make([]bool, len(es)) + for i, e := range es { + if e.op == ' ' { + continue + } + for j := max(0, i-context); j <= min(len(es)-1, i+context); j++ { + keep[j] = true + } + } + var b strings.Builder + fmt.Fprintf(&b, "--- %s\n+++ the policy now\n", filepath.ToSlash(path)) + for i := 0; i < len(es); { + if !keep[i] { + i++ + continue + } + end := i + for end < len(es) && keep[end] { + end++ + } + var olds, news int + for _, e := range es[i:end] { + if e.op != '+' { + olds++ + } + if e.op != '-' { + news++ + } + } + fmt.Fprintf(&b, "@@ -%d,%d +%d,%d @@\n", es[i].a+1, olds, es[i].b+1, news) + for _, e := range es[i:end] { + b.WriteByte(e.op) + b.WriteString(e.line) + b.WriteByte('\n') + } + i = end + } + return b.String() +} + +// edit is one line of a diff: kept (' '), removed ('-') or added ('+'), +// with the index each side has reached. +type edit struct { + op byte + line string + a, b int +} + +// edits is a shortest edit script from a to b, by longest common +// subsequence. Snapshots are small; past a few million cells it gives up +// on alignment and replaces a with b wholesale. +func edits(a, b []string) []edit { + var out []edit + if len(a)*len(b) > 1<<22 { + for i, l := range a { + out = append(out, edit{'-', l, i, 0}) + } + for j, l := range b { + out = append(out, edit{'+', l, len(a), j}) + } + return out + } + lcs := make([][]int32, len(a)+1) + for i := range lcs { + lcs[i] = make([]int32, len(b)+1) + } + for i := len(a) - 1; i >= 0; i-- { + for j := len(b) - 1; j >= 0; j-- { + if a[i] == b[j] { + lcs[i][j] = lcs[i+1][j+1] + 1 + } else { + lcs[i][j] = max(lcs[i+1][j], lcs[i][j+1]) + } + } + } + i, j := 0, 0 + for i < len(a) || j < len(b) { + switch { + case i < len(a) && j < len(b) && a[i] == b[j]: + out = append(out, edit{' ', a[i], i, j}) + i++ + j++ + case j == len(b) || i < len(a) && lcs[i+1][j] >= lcs[i][j+1]: + out = append(out, edit{'-', a[i], i, j}) + i++ + default: + out = append(out, edit{'+', b[j], i, j}) + j++ + } + } + return out +} + +// lines splits text into its lines, without the final newline's empty +// one. +func lines(text []byte) []string { + s := strings.TrimSuffix(string(text), "\n") + if s == "" { + return nil + } + return strings.Split(s, "\n") +} diff --git a/languages/golang/stackencrypt/plan/plantest/golden_test.go b/languages/golang/stackencrypt/plan/plantest/golden_test.go new file mode 100644 index 000000000..4d71d86b4 --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/golden_test.go @@ -0,0 +1,47 @@ +package plantest_test + +import ( + "testing" + + "github.com/cipherstash/stack/languages/golang/internal/factstest" + se "github.com/cipherstash/stack/languages/golang/stackencrypt" + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan/plantest" +) + +var category = plan.Key("fides.data_categories") + +type individual struct { + ID int64 + Email string `facts:"fides.data_categories=user.contact.email"` + Name string `facts:"fides.data_categories=user.name"` + MedicareNo string `facts:"fides.data_categories=user.government_id"` + Notes []byte `facts:"fides.data_categories=user.content"` + Country string `facts:"fides.data_categories=system.operations"` +} + +// individuals has a field whose database column was renamed (stored in +// medicare_num, under its first identity), a Custom target, and a field +// left in plaintext. +var individuals = plan.ForMessage(&individual{}, plan.Table("individuals"), plan.FirstOf( + plan.When(plan.Field("medicare_no"), plan.Encrypt(plan.EQL(se.Equality)), + plan.Column("medicare_num"), plan.Identity("medicare_number")), + plan.When(category.Under("user.content"), plan.Encrypt(plan.Custom("individuals-notes/v1"))), + plan.When(category.Under("user.contact.email"), plan.Encrypt(plan.EQL(se.Equality, se.Match))), + plan.When(category.Under("user"), plan.Encrypt(plan.EQL(se.Ore))), + plan.When(category.Under("system"), plan.Plaintext()), +)) + +type audit struct { + ID int64 + Kind string `facts:"fides.data_categories=system.operations"` +} + +// audits encrypts nothing: its snapshot is the fields it leaves plaintext. +var audits = plan.ForMessage(audit{}, plan.Table("audits"), plan.When(category.Under("system"), plan.Plaintext())) + +// Each message's snapshot is checked in at testdata/TestPolicies/.golden. +func TestPolicies(t *testing.T) { + t.Run("individuals", func(t *testing.T) { plantest.Golden(t, factstest.StructTags, individuals) }) + t.Run("audits", func(t *testing.T) { plantest.Golden(t, factstest.StructTags, audits) }) +} diff --git a/languages/golang/stackencrypt/plan/plantest/plantest.go b/languages/golang/stackencrypt/plan/plantest/plantest.go new file mode 100644 index 000000000..e673717c6 --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/plantest.go @@ -0,0 +1,185 @@ +// Package plantest checks in what a [plan.Policy] stores each field as, so +// a change to a field's encryption context fails a test instead of the +// data. +// +// A field's context is bound into every ciphertext and query term written +// under it. A policy derives contexts from names, so an ordinary rename — +// a proto field, a Go struct field, a table — can change one with no error +// on the write path: rows already written stop decrypting, and their terms +// stop matching queries. [Golden] makes that a test failure: +// +// func TestIndividualsPolicy(t *testing.T) { +// plantest.Golden(t, source, policy.Individuals) +// } +// +// The first run with -update writes the snapshot to +// testdata/TestIndividualsPolicy.golden; check it in. From then on Golden +// builds the plan the way [plan.MustPlanFor] does at startup, and fails +// when the plan no longer matches the snapshot, naming each change and +// calling out a changed context apart from the rest: a changed context is +// data loss, while a changed target (its index terms, or encrypting a +// field that was plaintext) is a migration. When a rename changed a +// context, the failure names the [plan.Column] (or [plan.Identity]) pin +// that keeps it, checked by building the plan with that pin. +// +// # The snapshot +// +// The snapshot records what a policy stores, not what the schema calls it: +// per message its [plan.Table]; per encrypted field its column (the record +// key), its context, its index terms and its facts; per field decided +// [plan.Plaintext], its name and its facts. Fields with no facts that no +// rule names are not the policy's concern and are left out. Encrypted +// fields are named by column, so a schema rename that the policy pins +// leaves the snapshot byte-for-byte unchanged, which is how a reviewer +// tells a safe rename from one that loses data. +// +// The text is line-oriented and deterministic across runs and platforms: +// fields are sorted, facts are sorted, line endings are "\n" (a checkout +// that converted them to "\r\n" still compares equal), and any value that +// is not a plain identifier is quoted as a Go string literal. It is meant +// to be read by people as well as compared: the list of classified fields, +// what protects each, and the identifier it is bound to. +// +// # The -update flag +// +// plantest registers the conventional -update test flag, as a bool on the +// default flag set, when no package it imports has done so already. A test +// package that defines its own -update panics with "flag redefined"; +// read plantest's instead, with flag.Lookup("update"). +package plantest + +import ( + "errors" + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" +) + +func init() { + if flag.Lookup("update") == nil { + flag.Bool("update", false, "rewrite plantest golden snapshots (testdata/*.golden) from the policy") + } +} + +// updating reports whether the test binary was run with -update. +func updating() bool { + f := flag.Lookup("update") + if f == nil { + return false + } + g, ok := f.Value.(flag.Getter) + if !ok { + return false + } + on, ok := g.Get().(bool) + return ok && on +} + +// Golden builds m's plan over src's facts, as [plan.PlanFor] does, and +// checks what it stores each field as against the snapshot checked in at +// testdata/.golden (a subtest's name is a path below +// testdata). With -update it writes the snapshot instead, logging what +// changed. +// +// The test fails when the policy does not build (a classified field no +// rule decides, a [plan.Fail], a plan that does not bind to the message), +// when there is no snapshot yet, and when the plan differs from it. A +// message the policy encrypts nothing of is not a failure: its snapshot +// lists the fields it decided Plaintext. +// +// A [plan.Source] is expected to be pure, as a policy is: Golden reads the +// facts more than once. +func Golden(t testing.TB, src plan.Source, m plan.Message) { + t.Helper() + logs, err := check(goldenPath(t.Name()), src, m, updating(), rerun(t.Name())) + if logs != "" { + t.Log(logs) + } + if err != nil { + t.Error(err) + } +} + +// goldenPath is the snapshot file for a test: testdata/.golden, a +// subtest's name a path below it. A character a file system could refuse +// is spelled '_'. +func goldenPath(name string) string { + parts := strings.Split(name, "/") + for i, p := range parts { + parts[i] = strings.Map(func(r rune) rune { + if r < 0x80 && (r == '-' || r == '_' || r == '.' || r == '+' || + 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || '0' <= r && r <= '9') { + return r + } + return '_' + }, p) + if strings.Trim(parts[i], ".") == "" { + parts[i] = strings.Repeat("_", len(parts[i])+1) + } + } + return filepath.Join(append([]string{"testdata"}, parts...)...) + ".golden" +} + +// rerun is the command that runs just this test with -update. +func rerun(name string) string { + parts := strings.Split(name, "/") + for i, p := range parts { + parts[i] = "^" + regexp.QuoteMeta(p) + "$" + } + return fmt.Sprintf("go test -run '%s' -update", strings.Join(parts, "/")) +} + +// check is Golden without the testing.TB: it compares (or, updating, +// writes) the snapshot at path, and returns what to log and what failed. +func check(path string, src plan.Source, m plan.Message, update bool, rerun string) (string, error) { + cur, facts, err := take(src, m) + if err != nil { + return "", fmt.Errorf("plantest: the policy does not build, so there is nothing to snapshot: %w", err) + } + got := cur.render() + path = filepath.Clean(path) + want, readErr := os.ReadFile(path) + if readErr != nil && !errors.Is(readErr, fs.ErrNotExist) { + return "", fmt.Errorf("plantest: %w", readErr) + } + same := readErr == nil && string(normalize(want)) == string(got) + + if update { + if same { + return "", nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return "", fmt.Errorf("plantest: %w", err) + } + // A snapshot is checked in, not secret; git records no mode but + // the executable bit. + if err := os.WriteFile(path, got, 0o644); err != nil { //nolint:gosec // see above + return "", fmt.Errorf("plantest: %w", err) + } + if readErr != nil { + return fmt.Sprintf("plantest: wrote %s; review it and check it in", path), nil + } + return fmt.Sprintf("plantest: updated %s, recording:\n\n%s", path, strings.TrimRight(explain(want, cur, facts, m), "\n")), nil + } + + if readErr != nil { + return "", fmt.Errorf("plantest: no snapshot at %s. Write it with\n\n\t%s\n\nthen review it and check it in", path, rerun) + } + if same { + return "", nil + } + return "", fmt.Errorf("plantest: %s no longer matches the policy.\n\n%sIf every change is intended (no row has been written under a changed context yet, or a migration ships with it), record them with\n\n\t%s\n\n%s", + path, explain(want, cur, facts, m), rerun, diff(path, normalize(want), got)) +} + +// normalize undoes a checkout's "\r\n" line endings. +func normalize(b []byte) []byte { + return []byte(strings.ReplaceAll(string(b), "\r\n", "\n")) +} diff --git a/languages/golang/stackencrypt/plan/plantest/plantest_internal_test.go b/languages/golang/stackencrypt/plan/plantest/plantest_internal_test.go new file mode 100644 index 000000000..20ba432a1 --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/plantest_internal_test.go @@ -0,0 +1,471 @@ +package plantest + +import ( + "bytes" + "errors" + "flag" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/internal/factstest" + se "github.com/cipherstash/stack/languages/golang/stackencrypt" + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" +) + +var category = plan.Key("fides.data_categories") + +var base = plan.FirstOf( + plan.When(category.Under("user.government_id"), plan.Encrypt(plan.EQL(se.Equality))), + plan.When(category.Under("user.contact.email"), plan.Encrypt(plan.EQL(se.Equality, se.Match))), + plan.When(category.Under("user"), plan.Encrypt(plan.EQL())), + plan.When(category.Under("system"), plan.Plaintext()), +) + +func classified(values ...string) []plan.Annotation { + return []plan.Annotation{{Key: "fides.data_categories", Values: values}} +} + +// proto is a protobuf-shaped source: the facts are given, and a field +// keeps its number through a rename. +func proto(facts ...plan.Fact) plan.Source { + for i := range facts { + facts[i].Message = "acme.v1.Individual" + } + return plan.SourceFunc(func(any) ([]plan.Fact, error) { return facts, nil }) +} + +// individualV1 is the schema before a rename. +func individualV1() plan.Source { + return proto( + plan.Fact{Field: "id", GoField: "Id", Number: 1, Kind: "int64"}, + plan.Fact{Field: "email", GoField: "Email", Number: 2, Kind: "string", Annotations: classified("user.contact.email")}, + plan.Fact{Field: "medicare_number", GoField: "MedicareNumber", Number: 3, Kind: "string", Annotations: classified("user.government_id")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Kind: "string", Annotations: classified("system.operations")}, + ) +} + +// individualV2 is individualV1 with field 3 renamed. +func individualV2() plan.Source { + return proto( + plan.Fact{Field: "id", GoField: "Id", Number: 1, Kind: "int64"}, + plan.Fact{Field: "email", GoField: "Email", Number: 2, Kind: "string", Annotations: classified("user.contact.email")}, + plan.Fact{Field: "medicare_no", GoField: "MedicareNo", Number: 3, Kind: "string", Annotations: classified("user.government_id")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Kind: "string", Annotations: classified("system.operations")}, + ) +} + +// record writes the snapshot of m over src to a fresh path and returns +// the path and what it wrote. +func record(t *testing.T, src plan.Source, m plan.Message) (string, []byte) { + t.Helper() + path := filepath.Join(t.TempDir(), "testdata", "TestPolicy.golden") + logs, err := check(path, src, m, true, "RERUN") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(logs, "wrote "+path) { + t.Errorf("logs = %q, want the path written", logs) + } + written, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return path, written +} + +func mustContain(t *testing.T, err error, wants ...string) { + t.Helper() + if err == nil { + t.Fatalf("passed, want a failure saying %q", wants) + } + for _, want := range wants { + if !strings.Contains(err.Error(), want) { + t.Errorf("failure does not say %q:\n%s", want, err) + } + } +} + +// The acceptance case: a renamed proto field with no pin fails with a +// context change, naming the pin; with the pin it passes, and the +// snapshot is the one checked in, byte for byte. +func TestRenameWithoutAPinIsAContextChange(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, before := record(t, individualV1(), m) + + _, err := check(path, individualV2(), m, false, "RERUN") + mustContain(t, err, + "CONTEXT CHANGES", + `column medicare_number: no field writes its context "individuals/medicare_number" any more.`, + `Field medicare_no (MedicareNo) now writes column medicare_no under "individuals/medicare_no"`, + `Pinning the rule that decides field medicare_no (MedicareNo) with plan.Column("medicare_number") keeps it.`, + "RERUN", + "- context individuals/medicare_number", + "+ context individuals/medicare_no", + ) + // A context change is data loss, not a migration or a new column. + if strings.Contains(err.Error(), "TARGET CHANGES") || strings.Contains(err.Error(), "OTHER CHANGES") { + t.Errorf("a rename is reported as more than a context change:\n%s", err) + } + + pinned := plan.ForMessage(nil, "individuals", plan.FirstOf( + plan.When(plan.Field("medicare_no"), plan.Encrypt(plan.EQL(se.Equality)), plan.Column("medicare_number")), + ).OrElse(base)) + if _, err := check(path, individualV2(), pinned, false, "RERUN"); err != nil { + t.Fatalf("the pinned rename fails: %v", err) + } + after, _, err := take(individualV2(), pinned) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after.render(), before) { + t.Fatalf("the pinned rename changed the snapshot:\n%s\nwas\n%s", after.render(), before) + } +} + +// The same holds for a Go struct field renamed, where a fact source has +// no field number and the plan binds to the struct. +func TestStructFieldRename(t *testing.T) { + type v1 struct { + ID int64 + MedicareNumber string `facts:"fides.data_categories=user.government_id"` + } + type v2 struct { + ID int64 + MedicareNo string `facts:"fides.data_categories=user.government_id"` + } + path, _ := record(t, factstest.StructTags, plan.ForMessage(v1{}, "individuals", base)) + _, err := check(path, factstest.StructTags, plan.ForMessage(v2{}, "individuals", base), false, "RERUN") + mustContain(t, err, "CONTEXT CHANGES", `plan.Column("medicare_number")`) + pinned := plan.ForMessage(v2{}, "individuals", plan.FirstOf( + plan.When(plan.Field("medicare_no"), plan.Encrypt(plan.EQL(se.Equality)), plan.Column("medicare_number")), + ).OrElse(base)) + if _, err := check(path, factstest.StructTags, pinned, false, "RERUN"); err != nil { + t.Fatalf("the pinned rename fails: %v", err) + } +} + +// Each kind of change lands in its own section, with the advice that +// fits it. +func TestChangesAreSortedByWhatTheyCost(t *testing.T) { + gov := plan.Encrypt(plan.EQL(se.Equality)) + for name, tc := range map[string]struct { + before, after plan.Message + src plan.Source // individualV1 when nil + section string // the only section reported; "" when it does not build + says []string + }{ + "identity pin dropped": { + before: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), gov, plan.Identity("medicare_no"))).OrElse(base)), + after: plan.ForMessage(nil, "individuals", base), + section: "CONTEXT CHANGES", + says: []string{`column medicare_number: its context is "individuals/medicare_number", was "individuals/medicare_no".`, `with plan.Identity("medicare_no") keeps it.`}, + }, + "table changed": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "people", base), + section: "CONTEXT CHANGES", + says: []string{`the message's table is people, was individuals.`, `restore plan.Table("individuals")`, `its context is "people/email", was "individuals/email". Restoring plan.Table("individuals") brings it back.`}, + }, + "custom context changed": { + before: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), plan.Encrypt(plan.Custom("gov/v1")))).OrElse(base)), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), plan.Encrypt(plan.Custom("gov/v2")))).OrElse(base)), + section: "CONTEXT CHANGES", + says: []string{`its context is "gov/v2", was "gov/v1". No plan.Column or plan.Identity pin`, "a plan.Custom context"}, + }, + "encrypted now plaintext": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), plan.Plaintext())).OrElse(base)), + section: "TARGET CHANGES", + says: []string{"column medicare_number is now field medicare_number, decided Plaintext.", `ciphertexts under "individuals/medicare_number"`}, + }, + "terms changed": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("email"), plan.Encrypt(plan.EQL(se.Equality)))).OrElse(base)), + section: "TARGET CHANGES", + says: []string{"column email: its terms are [eq], were [eq match]."}, + }, + "plaintext now encrypted": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("country"), plan.Encrypt(plan.EQL()))).OrElse(base)), + section: "TARGET CHANGES", + says: []string{`field country, decided Plaintext before, is now encrypted into column country under "individuals/country".`}, + }, + "database column renamed": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), gov, plan.Column("medicare_num"), plan.Identity("medicare_number"))).OrElse(base)), + section: "TARGET CHANGES", + says: []string{"column medicare_number is now stored in column medicare_num, under the same context"}, + }, + "field newly classified": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("id"), plan.Encrypt(plan.EQL(se.Ore)))).OrElse(base)), + section: "OTHER CHANGES", + says: []string{`new column id, under "individuals/id", with terms [ore].`}, + }, + "facts changed": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", base), + src: proto( + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: classified("user.contact.email", "user.contact.email.work")}, + plan.Fact{Field: "medicare_number", GoField: "MedicareNumber", Number: 3, Annotations: classified("user.government_id")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Annotations: classified("system.operations", "system.location")}, + ), + section: "OTHER CHANGES", + says: []string{ + "column email: its facts are [fides.data_categories=user.contact.email fides.data_categories=user.contact.email.work], were [fides.data_categories=user.contact.email].", + "plaintext field country: its facts are", + }, + }, + "plaintext field gone": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", base), + src: proto( + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: classified("user.contact.email")}, + plan.Fact{Field: "medicare_number", GoField: "MedicareNumber", Number: 3, Annotations: classified("user.government_id")}, + ), + section: "OTHER CHANGES", + says: []string{"plaintext field country is no longer decided by the policy"}, + }, + "refused": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("country"), plan.Fail("never stored"))).OrElse(base)), + says: []string{"the policy does not build", "never stored"}, + }, + } { + t.Run(name, func(t *testing.T) { + path, _ := record(t, individualV1(), tc.before) + src := tc.src + if src == nil { + src = individualV1() + } + _, err := check(path, src, tc.after, false, "RERUN") + mustContain(t, err, tc.says...) + if tc.section == "" { + return + } + for _, s := range []string{"CONTEXT CHANGES", "TARGET CHANGES", "OTHER CHANGES"} { + if got := strings.Contains(err.Error(), s); got != (s == tc.section) { + t.Errorf("reports %s = %v, want only %s:\n%s", s, got, tc.section, err) + } + } + }) + } +} + +// A field removed from the schema without a trace is a lost context: the +// failure says what to do either way. +func TestALostContextWithNoCandidate(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, _ := record(t, individualV1(), m) + gone := proto( + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: classified("user.contact.email")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Annotations: classified("system.operations")}, + ) + _, err := check(path, gone, m, false, "RERUN") + mustContain(t, err, "CONTEXT CHANGES", `column medicare_number: no field writes its context "individuals/medicare_number" any more. If its field was renamed, pin the renamed field's rule with plan.Column("medicare_number")`) +} + +// Two new columns with the facts of the one that disappeared: no guess, +// and the generic advice. +func TestAmbiguousRenameIsNotGuessed(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, _ := record(t, individualV1(), m) + split := proto( + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: classified("user.contact.email")}, + plan.Fact{Field: "medicare_a", GoField: "MedicareA", Number: 5, Annotations: classified("user.government_id")}, + plan.Fact{Field: "medicare_b", GoField: "MedicareB", Number: 6, Annotations: classified("user.government_id")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Annotations: classified("system.operations")}, + ) + _, err := check(path, split, m, false, "RERUN") + mustContain(t, err, "If its field was renamed", "new column medicare_a", "new column medicare_b") + if strings.Contains(err.Error(), "likely the same field renamed") { + t.Errorf("guessed a rename between two candidates:\n%s", err) + } +} + +// The snapshot is the same whatever order the source gives fields and +// facts in, and from run to run. +func TestSnapshotIsDeterministic(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + a, _, err := take(individualV1(), m) + if err != nil { + t.Fatal(err) + } + shuffled := proto( + plan.Fact{Field: "country", GoField: "Country", Number: 4, Annotations: classified("system.operations")}, + plan.Fact{Field: "medicare_number", GoField: "MedicareNumber", Number: 3, Annotations: classified("user.government_id")}, + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: []plan.Annotation{ + {Key: "fides.data_categories", Values: []string{"user.contact.email"}}, + {Key: "fides.data_categories", Values: []string{"user.contact.email"}}, + }}, + plan.Fact{Field: "id", GoField: "Id", Number: 1}, + ) + b, _, err := take(shuffled, m) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(a.render(), b.render()) { + t.Fatalf("field or fact order changed the snapshot:\n%s\nvs\n%s", a.render(), b.render()) + } + want := header + ` +table individuals + +column email + context individuals/email + terms eq match + fact fides.data_categories user.contact.email + +column medicare_number + context individuals/medicare_number + terms eq + fact fides.data_categories user.government_id + +plaintext country + fact fides.data_categories system.operations +` + if got := string(a.render()); got != want { + t.Fatalf("snapshot =\n%s\nwant\n%s", got, want) + } +} + +// Whatever a name, a context or a fact holds, the snapshot reads back as +// what was written. +func TestSnapshotRoundTrips(t *testing.T) { + odd := []string{"", "with space", `"quoted"`, "new\nline", "tab\there", "naïve", "#hash", "none", "a=b,c"} + s := snapshot{table: "odd table"} + for i, v := range odd { + s.columns = append(s.columns, column{name: v + string(rune('a'+i)), context: v, terms: []string{"eq", "ore"}, facts: []fact{{v, v}, {"k", v}}}) + s.plaintext = append(s.plaintext, plain{field: v + string(rune('a'+i)), facts: []fact{{v, "x"}}}) + } + s.columns = append(s.columns, column{name: "bare", context: "t/bare"}) + for i := range s.columns { + sortFacts(s.columns[i].facts) + } + s.sort() + text := s.render() + back, err := parse(text) + if err != nil { + t.Fatalf("%v\n%s", err, text) + } + if again := back.render(); !bytes.Equal(again, text) { + t.Fatalf("round trip changed the snapshot:\n%s\nwas\n%s", again, text) + } + if !strings.Contains(string(text), " terms none\n") { + t.Errorf("an unindexed column does not say so:\n%s", text) + } +} + +func TestMissingSnapshot(t *testing.T) { + path := filepath.Join(t.TempDir(), "testdata", "TestPolicy.golden") + _, err := check(path, individualV1(), plan.ForMessage(nil, "individuals", base), false, "go test -run '^TestPolicy$' -update") + mustContain(t, err, "no snapshot at "+path, "go test -run '^TestPolicy$' -update") +} + +func TestUpdateRewritesAndSaysWhatItRecorded(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, _ := record(t, individualV1(), m) + logs, err := check(path, individualV2(), m, true, "RERUN") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(logs, "updated "+path) || !strings.Contains(logs, "CONTEXT CHANGES") { + t.Errorf("logs = %q, want the update and the context change it recorded", logs) + } + if _, err := check(path, individualV2(), m, false, "RERUN"); err != nil { + t.Fatalf("after -update: %v", err) + } + // Updating an unchanged snapshot is silent and leaves it alone. + if logs, err := check(path, individualV2(), m, true, "RERUN"); err != nil || logs != "" { + t.Errorf("no-op update: logs %q, err %v", logs, err) + } +} + +// A checkout that converted line endings still matches. +func TestCRLFCheckoutMatches(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, written := record(t, individualV1(), m) + if err := os.WriteFile(path, bytes.ReplaceAll(written, []byte("\n"), []byte("\r\n")), 0o600); err != nil { + t.Fatal(err) + } + if _, err := check(path, individualV1(), m, false, "RERUN"); err != nil { + t.Fatal(err) + } +} + +func TestTextOnlyAndUnreadableSnapshots(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, written := record(t, individualV1(), m) + + // Same content, different text: still a failure, said as such. + edited := bytes.Replace(written, []byte("table individuals"), []byte(`table "individuals"`), 1) + if err := os.WriteFile(path, edited, 0o600); err != nil { + t.Fatal(err) + } + _, err := check(path, individualV1(), m, false, "RERUN") + mustContain(t, err, "Nothing the policy stores changed", `-table "individuals"`, "+table individuals") + + if err := os.WriteFile(path, append(written, []byte("garbage line\n")...), 0o600); err != nil { + t.Fatal(err) + } + _, err = check(path, individualV1(), m, false, "RERUN") + mustContain(t, err, "does not parse", `unexpected "garbage line"`, "+++ the policy now") +} + +// A policy that does not build fails the test with the build's error. +func TestPolicyThatDoesNotBuild(t *testing.T) { + narrow := plan.ForMessage(nil, "individuals", plan.When(category.Under("user.contact"), plan.Encrypt(plan.EQL()))) + _, err := check(filepath.Join(t.TempDir(), "x.golden"), individualV1(), narrow, true, "RERUN") + if !errors.Is(err, plan.ErrUnmatched) { + t.Fatalf("err = %v, want ErrUnmatched", err) + } + mustContain(t, err, "the policy does not build", "medicare_number") + if _, err := check("x.golden", nil, narrow, true, "RERUN"); err == nil || !strings.Contains(err.Error(), "needs a Source") { + t.Errorf("nil source: %v", err) + } +} + +// A message the policy encrypts nothing of has a snapshot: the fields it +// decided Plaintext. +func TestNothingEncrypted(t *testing.T) { + m := plan.ForMessage(nil, "individuals", plan.When(category.Present(), plan.Plaintext())) + _, written := record(t, individualV1(), m) + if strings.Contains(string(written), "column ") || !strings.Contains(string(written), "plaintext medicare_number") { + t.Fatalf("snapshot:\n%s", written) + } +} + +func TestGoldenPathAndRerun(t *testing.T) { + for name, want := range map[string]string{ + "TestPolicy": filepath.Join("testdata", "TestPolicy.golden"), + "TestPolicy/individuals": filepath.Join("testdata", "TestPolicy", "individuals.golden"), + "TestPolicy/a:b*c?": filepath.Join("testdata", "TestPolicy", "a_b_c_.golden"), + "TestPolicy/..": filepath.Join("testdata", "TestPolicy", "___.golden"), + "TestPolicy/v1.2+build_name-x": filepath.Join("testdata", "TestPolicy", "v1.2+build_name-x.golden"), + } { + if got := goldenPath(name); got != want { + t.Errorf("goldenPath(%q) = %q, want %q", name, got, want) + } + } + if got, want := rerun("TestPolicy/a.b"), `go test -run '^TestPolicy$/^a\.b$' -update`; got != want { + t.Errorf("rerun = %s, want %s", got, want) + } +} + +func TestUpdateFlag(t *testing.T) { + f := flag.Lookup("update") + if f == nil { + t.Fatal("-update is not registered") + } + if updating() { + t.Skip("run with -update") + } + if err := f.Value.Set("true"); err != nil { + t.Fatal(err) + } + defer func() { _ = f.Value.Set("false") }() + if !updating() { + t.Error("updating() does not follow -update") + } +} diff --git a/languages/golang/stackencrypt/plan/plantest/snapshot.go b/languages/golang/stackencrypt/plan/plantest/snapshot.go new file mode 100644 index 000000000..1cc8a6135 --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/snapshot.go @@ -0,0 +1,282 @@ +package plantest + +import ( + "bytes" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/cipherstash/stack/languages/golang/stackencrypt" + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" +) + +// header opens every snapshot. One sentence per line: the file is read in +// diffs, where a rewrapped paragraph would read as a change. +const header = `# Written by plantest.Golden: what the policy stores each field it decides as. +# Regenerate it with go test -update; do not edit it by hand. +# A column's context is bound into every ciphertext and query term written under it, so once a row is written it must never change. +` + +// snapshot is what a message's policy stores each field it decides as: +// the persistent half of a plan, which a rename the policy pins leaves +// alone. +type snapshot struct { + table string + columns []column // encrypted fields, by column + plaintext []plain // fields decided Plaintext, by field +} + +// column is one encrypted field, named by its record key. +type column struct { + name string + context string + terms []string + facts []fact + // from is the field it is decided from now; nil when read from a file. + from *decided +} + +// plain is one field decided Plaintext, named by its schema name. +type plain struct { + field string + facts []fact + from *decided +} + +// decided is a field of the message as the policy sees it now. +type decided struct { + fact plan.Fact + decision plan.Decision +} + +// fact is one annotation value. +type fact struct{ key, value string } + +// take builds m's plan from src, as plan.PlanFor does at startup, and +// snapshots it. It returns the facts as well, for the hints a comparison +// builds. +func take(src plan.Source, m plan.Message) (snapshot, []plan.Fact, error) { + if src == nil { + return snapshot{}, nil, errors.New("plantest: Golden needs a Source") + } + p, err := plan.PlanFor(src, m) + if err != nil && !errors.Is(err, plan.ErrNothingEncrypted) { + return snapshot{}, nil, err + } + facts, err := src.Facts(m.Msg()) + if err != nil { + return snapshot{}, nil, err + } + planned := map[string]stackencrypt.FieldPlan{} + for _, fp := range p.Fields() { + planned[fp.Field] = fp + } + s := snapshot{table: string(m.Table())} + seen := map[string]bool{} + for _, f := range facts { + d, ok := m.Decide(f) + if !ok { + // Unclassified and unnamed: PlanFor has already refused a + // classified field no rule decides. + continue + } + if seen[f.Field] { + return snapshot{}, nil, fmt.Errorf("plantest: the source gives field %q twice", f.Field) + } + seen[f.Field] = true + from := &decided{fact: f, decision: d} + if _, encrypted := d.Target(); !encrypted { + s.plaintext = append(s.plaintext, plain{field: f.Field, facts: factsOf(f), from: from}) + continue + } + fp, ok := planned[goField(f)] + if !ok { + return snapshot{}, nil, fmt.Errorf("plantest: %s is decided %v but is not in the plan", f, d) + } + terms := make([]string, len(fp.Terms)) + for i, k := range fp.Terms { + terms[i] = k.String() + } + s.columns = append(s.columns, column{name: fp.Name, context: fp.Context, terms: terms, facts: factsOf(f), from: from}) + } + s.sort() + return s, facts, nil +} + +// goField is the Go field a fact binds to, as the plan package spells it. +func goField(f plan.Fact) string { + if f.GoField != "" { + return f.GoField + } + return f.Field +} + +// factsOf is a field's annotations as sorted, distinct (key, value) pairs: +// their order in the schema means nothing to a policy. +func factsOf(f plan.Fact) []fact { + var out []fact + for _, a := range f.Annotations { + for _, v := range a.Values { + out = append(out, fact{a.Key, v}) + } + } + sortFacts(out) + return slices.Compact(out) +} + +func sortFacts(fs []fact) { + slices.SortFunc(fs, func(a, b fact) int { + if c := strings.Compare(a.key, b.key); c != 0 { + return c + } + return strings.Compare(a.value, b.value) + }) +} + +func (s *snapshot) sort() { + slices.SortFunc(s.columns, func(a, b column) int { return strings.Compare(a.name, b.name) }) + slices.SortFunc(s.plaintext, func(a, b plain) int { return strings.Compare(a.field, b.field) }) +} + +// render spells the snapshot. parse reads it back. +func (s snapshot) render() []byte { + var b bytes.Buffer + b.WriteString(header) + fmt.Fprintf(&b, "\ntable %s\n", token(s.table)) + for _, c := range s.columns { + fmt.Fprintf(&b, "\ncolumn %s\n", token(c.name)) + fmt.Fprintf(&b, " context %s\n", token(c.context)) + fmt.Fprintf(&b, " terms %s\n", termList(c.terms)) + writeFacts(&b, c.facts) + } + for _, p := range s.plaintext { + fmt.Fprintf(&b, "\nplaintext %s\n", token(p.field)) + writeFacts(&b, p.facts) + } + return b.Bytes() +} + +// termList spells a column's terms, "none" for none: an unindexed column +// is a decision too, and says nothing about the value but its ciphertext. +func termList(terms []string) string { + if len(terms) == 0 { + return "none" + } + return strings.Join(terms, " ") +} + +func writeFacts(b *bytes.Buffer, facts []fact) { + for _, f := range facts { + fmt.Fprintf(b, " fact %s %s\n", token(f.key), token(f.value)) + } +} + +// token spells a value: bare when it is a plain identifier, a Go string +// literal otherwise, so a space, a quote or a newline in a name or a +// context cannot change how the line reads. +func token(s string) string { + if s == "" { + return strconv.Quote(s) + } + for i := 0; i < len(s); i++ { + if !bare(s[i]) { + return strconv.Quote(s) + } + } + return s +} + +// bare reports whether c may appear in an unquoted value. +func bare(c byte) bool { + return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' || + c == '_' || c == '.' || c == '/' || c == '-' || c == ':' || c == '@' || c == '+' +} + +// parse reads a snapshot render wrote. It is strict about structure and +// lenient about nothing else: any byte difference fails the comparison +// anyway, and parse only lets the failure say what changed. +func parse(data []byte) (snapshot, error) { + var s snapshot + var table bool + // The entry the indented lines below belong to: a column (>= 0), a + // plaintext field (plainAt >= 0), or neither. + colAt, plainAt := -1, -1 + for n, line := range strings.Split(string(normalize(data)), "\n") { + if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") { + continue + } + indented := strings.HasPrefix(line, " ") + toks, err := tokens(strings.TrimSpace(line)) + if err != nil { + return snapshot{}, fmt.Errorf("line %d: %w", n+1, err) + } + switch { + case !indented && toks[0] == "table" && len(toks) == 2 && !table: + s.table, table = toks[1], true + case !indented && toks[0] == "column" && len(toks) == 2: + s.columns = append(s.columns, column{name: toks[1]}) + colAt, plainAt = len(s.columns)-1, -1 + case !indented && toks[0] == "plaintext" && len(toks) == 2: + s.plaintext = append(s.plaintext, plain{field: toks[1]}) + colAt, plainAt = -1, len(s.plaintext)-1 + case indented && toks[0] == "context" && len(toks) == 2 && colAt >= 0: + s.columns[colAt].context = toks[1] + case indented && toks[0] == "terms" && len(toks) >= 2 && colAt >= 0: + if len(toks) == 2 && toks[1] == "none" { + s.columns[colAt].terms = nil + } else { + s.columns[colAt].terms = toks[1:] + } + case indented && toks[0] == "fact" && len(toks) == 3 && colAt >= 0: + s.columns[colAt].facts = append(s.columns[colAt].facts, fact{toks[1], toks[2]}) + case indented && toks[0] == "fact" && len(toks) == 3 && plainAt >= 0: + s.plaintext[plainAt].facts = append(s.plaintext[plainAt].facts, fact{toks[1], toks[2]}) + default: + return snapshot{}, fmt.Errorf("line %d: unexpected %q", n+1, line) + } + } + if !table { + return snapshot{}, errors.New("no table line") + } + for i := range s.columns { + sortFacts(s.columns[i].facts) + } + for i := range s.plaintext { + sortFacts(s.plaintext[i].facts) + } + s.sort() + return s, nil +} + +// tokens splits a line into its values, unquoting the quoted ones. +func tokens(line string) ([]string, error) { + var out []string + for line != "" { + if line[0] == '"' { + q, err := strconv.QuotedPrefix(line) + if err != nil { + return nil, fmt.Errorf("bad quoted value in %q", line) + } + v, err := strconv.Unquote(q) + if err != nil { + return nil, fmt.Errorf("bad quoted value in %q", line) + } + out = append(out, v) + line = line[len(q):] + } else { + end := strings.IndexByte(line, ' ') + if end < 0 { + end = len(line) + } + out = append(out, line[:end]) + line = line[end:] + } + line = strings.TrimLeft(line, " ") + } + if len(out) == 0 { + return nil, errors.New("empty line") + } + return out, nil +} diff --git a/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/audits.golden b/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/audits.golden new file mode 100644 index 000000000..7c0ccce9f --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/audits.golden @@ -0,0 +1,8 @@ +# Written by plantest.Golden: what the policy stores each field it decides as. +# Regenerate it with go test -update; do not edit it by hand. +# A column's context is bound into every ciphertext and query term written under it, so once a row is written it must never change. + +table audits + +plaintext kind + fact fides.data_categories system.operations diff --git a/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/individuals.golden b/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/individuals.golden new file mode 100644 index 000000000..56e39a3ff --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/individuals.golden @@ -0,0 +1,28 @@ +# Written by plantest.Golden: what the policy stores each field it decides as. +# Regenerate it with go test -update; do not edit it by hand. +# A column's context is bound into every ciphertext and query term written under it, so once a row is written it must never change. + +table individuals + +column email + context individuals/email + terms eq match + fact fides.data_categories user.contact.email + +column medicare_num + context individuals/medicare_number + terms eq + fact fides.data_categories user.government_id + +column name + context individuals/name + terms ore + fact fides.data_categories user.name + +column notes + context individuals-notes/v1 + terms none + fact fides.data_categories user.content + +plaintext country + fact fides.data_categories system.operations