diff --git a/languages/golang/stackencrypt/README.md b/languages/golang/stackencrypt/README.md index bef131e87..824ec32fe 100644 --- a/languages/golang/stackencrypt/README.md +++ b/languages/golang/stackencrypt/README.md @@ -296,6 +296,51 @@ their record key, and `plan.Identity` is refused. The plan a policy builds is a `Plan` like any other: the guest receives the same bytes as for the equivalent hand-built plan. +### Checking contexts in with a golden test + +Nothing on the write path notices a changed context: rename a proto or +struct field with no pin and new rows are simply written under a new one, +while the rows already written stop decrypting. The `plan/plantest` package +turns that into a test failure: + +```go +import "github.com/cipherstash/stack/languages/golang/stackencrypt/plan/plantest" + +func TestIndividualsPolicy(t *testing.T) { + plantest.Golden(t, source, Individuals) +} +``` + +Run it once with `go test -run '^TestIndividualsPolicy$' -update` to write +`testdata/TestIndividualsPolicy.golden`, and check the file in. It lists the +message's table and, for every field the policy decides, what it is stored +as: an encrypted field's column, context, index terms and facts, or a +plaintext field's name and facts. + +```text +table individuals + +column email + context individuals/email + terms eq match + fact fides.data_categories user.contact.email + +column medicare_number + context individuals/medicare_number + terms eq + fact fides.data_categories user.government_id +``` + +From then on the test builds the plan as `MustPlanFor` does at startup and +fails when it no longer matches the file, sorting the changes by what they +cost. A changed context is data loss and is reported first, with the +`plan.Column` (or `plan.Identity`) pin that keeps it; a changed target, such +as different index terms or a plaintext field now encrypted, is a migration; +anything else, such as a new field, is reported last. Encrypted fields are +listed by column, not by field name, so a rename the policy pins leaves the +file unchanged and the test passes. When a change is intended, rerun with +`-update` and review the diff. + ## Errors Errors are sentinel values, matched with `errors.Is`. The wasm guest diff --git a/languages/golang/stackencrypt/cipher.go b/languages/golang/stackencrypt/cipher.go index 5820a6aa8..79fbc6bad 100644 --- a/languages/golang/stackencrypt/cipher.go +++ b/languages/golang/stackencrypt/cipher.go @@ -86,13 +86,28 @@ func (cph *Cipher) DecryptElement(ctx context.Context, ct any, aad []byte) (any, // encoding); a string for Match; any scalar for Ore and Ope. The result is // one of EqualityTerm, MatchTerm, OreTerm or OpeTerm. // +// opts are [Option]s, the options a probe shares with the record calls; a +// [RecordOption] that only a record call takes, such as [WithPlan], does +// not compile here. [ExtendContext] extends context exactly as it extends +// each field's own context in a record call, so a probe for a field +// written under an extension is the field's context plus the same option +// value the rows were written with, never a context spelled by hand. +// // Term takes a context and returns an error because it may be a ZeroKMS // round trip: term derivation is asynchronous in the Rust crate, and a // ZeroKMS backend that derives terms server-side settles the same way. -func (cph *Cipher) Term(ctx context.Context, value any, context Context, kind TermKind) (any, error) { +func (cph *Cipher) Term(ctx context.Context, value any, context Context, kind TermKind, opts ...Option) (any, error) { if context.node == nil { return nil, fmt.Errorf("stackencrypt: term context is empty") } + var o termOptions + for _, opt := range opts { + opt.applyTerm(&o) + } + context, err := extend(context, o.extension) + if err != nil { + return nil, err + } encodedValue, err := vcffi.Marshal(value) if err != nil { return nil, err @@ -105,12 +120,12 @@ func (cph *Cipher) Term(ctx context.Context, value any, context Context, kind Te return nil, err } defer wipe(encodedContext) - opts, err := vcffi.Marshal(options(cph.keyset)) + encodedOpts, err := vcffi.Marshal(options(cph.keyset)) if err != nil { return nil, err } out, err := cph.client.call(ctx, func(inst *instance) ([]byte, error) { - return inst.call(ctx, inst.term, buf(encodedValue), buf(encodedContext), scalar(uint64(kind)), buf(opts)) + return inst.call(ctx, inst.term, buf(encodedValue), buf(encodedContext), scalar(uint64(kind)), buf(encodedOpts)) }) if err != nil { return nil, err diff --git a/languages/golang/stackencrypt/context.go b/languages/golang/stackencrypt/context.go index 4b8c6e750..49c968de1 100644 --- a/languages/golang/stackencrypt/context.go +++ b/languages/golang/stackencrypt/context.go @@ -1,6 +1,7 @@ package stackencrypt import ( + "bytes" "errors" "fmt" ) @@ -20,6 +21,9 @@ import ( // left, so NewContext("users/age").With(uint64(7)) is the context a row // sealed with encrypt_into_with_context(row, 7u64) binds for that field. // A one-element list is not the bare part, and this type cannot spell one. +// +// A Context owns its parts: a byte-slice part is copied in, so a caller's +// buffer reused once the Context is built does not change it. type Context struct { node any } @@ -41,7 +45,7 @@ func NewContext(part any) (Context, error) { if err := checkRootNonEmpty(part); err != nil { return Context{}, err } - return Context{node: part}, nil + return Context{node: ownPart(part)}, nil } // MustContext is [NewContext] for a part known to be valid; it panics @@ -63,7 +67,17 @@ func (c Context) With(part any) (Context, error) { if err := checkPart(part); err != nil { return Context{}, err } - return Context{node: []any{c.node, part}}, nil + return Context{node: []any{c.node, ownPart(part)}}, nil +} + +// ownPart is part as a context stores it: a byte slice is copied, so +// neither a Context nor an option that extends one ([ExtendContext]) +// aliases a caller's buffer. Every other part type is a value. +func ownPart(part any) any { + if b, ok := part.([]byte); ok { + return bytes.Clone(b) + } + return part } // value renders the context in the guest's grammar: a scalar or nested diff --git a/languages/golang/stackencrypt/doc.go b/languages/golang/stackencrypt/doc.go index 3bc5dd0af..fa22f8ea1 100644 --- a/languages/golang/stackencrypt/doc.go +++ b/languages/golang/stackencrypt/doc.go @@ -51,7 +51,13 @@ // for any ordinary value or batch, one more per 500 sealed leaves beyond // that. Terms ([EqualityTerm], [MatchTerm], [OreTerm], [OpeTerm]) are // byte-equal to the ones the Rust crate derives, so a probe from -// [Cipher.Term] compares against a stored term from any language. +// [Cipher.Term] compares against a stored term from any language. An +// [Option] is the one value that serves encrypt, decrypt and probe alike; +// a [RecordOption], such as [WithPlan], is what only a record call takes. +// [ExtendContext] is an Option: given to a record call and to the probe it +// extends the field's context and the probe's identically, so a +// tenant-scoped probe is the field's own context plus the option value the +// rows were written with, never a context spelled by hand. // [Cipher.Term] takes a context and returns an error from day one: term // derivation may be a ZeroKMS round trip. // diff --git a/languages/golang/stackencrypt/guest_test.go b/languages/golang/stackencrypt/guest_test.go index c876b17e9..9c54e03de 100644 --- a/languages/golang/stackencrypt/guest_test.go +++ b/languages/golang/stackencrypt/guest_test.go @@ -677,7 +677,11 @@ func TestGuestAcceptsEveryEncodingThisPackageBuilds(t *testing.T) { _, err := byID.Term(ctx, 1.5, c, Ore) return err }, - "Term ope bytes": func() error { _, err := def.Term(ctx, []byte{1}, MustContext("k"), Ope); return err }, + "Term ope bytes": func() error { _, err := def.Term(ctx, []byte{1}, MustContext("k"), Ope); return err }, + "Term ext option": func() error { + _, err := byID.Term(ctx, 1.5, MustContext("users/age"), Ore, ExtendContext(uint64(7), "eu")) + return err + }, "EncryptRecords": func() error { _, err := def.EncryptRecords(ctx, rows); return err }, "EncryptRecords ext": func() error { _, err := named.EncryptRecords(ctx, &rows, ExtendContext(uint64(7), "eu")); return err }, "EncryptRecord": func() error { _, err := byID.EncryptRecord(ctx, rows[0]); return err }, @@ -731,6 +735,22 @@ func TestGuestRefusesMalformedInputsBeforeState(t *testing.T) { "record without c": func() error { return c.DecryptRecord(ctx, EncryptedRecord{"Age": {Equality: EqualityTerm{1}}, "Email": {Ciphertext: Sealed(fixtureLeaf)}}, new(recordRow)) }, + // ExtendContext checks nothing when it is built; Context.With + // refuses the part when a call applies it. A call that dropped + // that error would run under a context missing the extension: + // a probe that matches no rows, or rows no probe matches. + "bad ext part in a term": func() error { + _, err := def.Term(ctx, 1, MustContext("k"), Equality, ExtendContext(1.5)) + return err + }, + "bad ext part in a record write": func() error { + _, err := def.EncryptRecords(ctx, []recordRow{{Age: 1, Email: "a@b.c"}}, ExtendContext(1.5)) + return err + }, + "bad ext part in a record read": func() error { + record := EncryptedRecord{"Age": {Ciphertext: Sealed(fixtureLeaf)}, "Email": {Ciphertext: Sealed(fixtureLeaf)}} + return c.DecryptRecord(ctx, record, new(recordRow), ExtendContext(1.5)) + }, } for name, call := range calls { err := call() @@ -742,6 +762,34 @@ func TestGuestRefusesMalformedInputsBeforeState(t *testing.T) { } } +// A bad ExtendContext part fails the call for that reason, on the probe and +// on both record directions. TestGuestRefusesMalformedInputsBeforeState +// shows the call never reaches the cipher, but a call that ignored the +// error and went on with an empty context would be refused too, by the +// guest, for another reason; only the error's own words tell the two apart. +func TestBadExtensionPartFailsTheCall(t *testing.T) { + ctx := context.Background() + c := rawInstance(t) + def := c.DefaultKeyset() + bad := ExtendContext(uint64(7), 1.5) + record := EncryptedRecord{"Age": {Ciphertext: Sealed(fixtureLeaf)}, "Email": {Ciphertext: Sealed(fixtureLeaf)}} + for name, call := range map[string]func() error{ + "Term": func() error { + _, err := def.Term(ctx, 1, MustContext("k"), Equality, bad) + return err + }, + "EncryptRecords": func() error { + _, err := def.EncryptRecords(ctx, []recordRow{{Age: 1, Email: "a@b.c"}}, bad) + return err + }, + "DecryptRecord": func() error { return c.DecryptRecord(ctx, record, new(recordRow), bad) }, + } { + if err := call(); err == nil || !strings.Contains(err.Error(), "float64 is not a context part") { + t.Errorf("%s with a float64 extension part: %v, want the part refused", name, err) + } + } +} + func TestClosedClientIsState(t *testing.T) { ctx := context.Background() c := rawInstance(t) diff --git a/languages/golang/stackencrypt/live_test.go b/languages/golang/stackencrypt/live_test.go index 9a49bee12..f3e4e50d2 100644 --- a/languages/golang/stackencrypt/live_test.go +++ b/languages/golang/stackencrypt/live_test.go @@ -220,6 +220,34 @@ func TestLiveRecordsAndTerms(t *testing.T) { if err := cipher.DecryptRecords(ctx, ext, &back, ExtendContext(uint64(7))); err != nil { t.Fatalf("extended record with its extension: %v", err) } + + // A probe takes the same option, and matches only the rows written + // under it: not another tenant's, and not the unextended ones. + tenant7, tenant8 := ExtendContext(uint64(7)), ExtendContext(uint64(8)) + other, err := cipher.EncryptRecords(ctx, users, tenant8) + if err != nil { + t.Fatal(err) + } + scoped, err := cipher.Term(ctx, "bob@example.com", MustContext("users/email"), Equality, tenant7) + if err != nil { + t.Fatal(err) + } + if !scoped.(EqualityTerm).Equal(ext[1]["Email"].Equality) { + t.Error("tenant probe does not equal the term written under the same extension") + } + if scoped.(EqualityTerm).Equal(other[1]["Email"].Equality) { + t.Error("tenant probe equals another tenant's term") + } + if scoped.(EqualityTerm).Equal(records[1]["Email"].Equality) { + t.Error("tenant probe equals the unextended term") + } + unscoped, err := cipher.Term(ctx, "bob@example.com", MustContext("users/email"), Equality) + if err != nil { + t.Fatal(err) + } + if unscoped.(EqualityTerm).Equal(ext[1]["Email"].Equality) { + t.Error("an unextended probe equals a tenant's term") + } } // An explicit plan round-trips a struct that carries no tags, and a record diff --git a/languages/golang/stackencrypt/plan/doc.go b/languages/golang/stackencrypt/plan/doc.go index 385da3a99..3614c8457 100644 --- a/languages/golang/stackencrypt/plan/doc.go +++ b/languages/golang/stackencrypt/plan/doc.go @@ -67,6 +67,16 @@ // A [Custom] target supplies its context itself; [Column] names only its // record key, and [Identity] is refused. // +// Nothing on the write path notices a context that changed: a rename with +// no pin simply writes new rows under a new context. Check them in with a +// golden test ([github.com/cipherstash/stack/languages/golang/stackencrypt/plan/plantest.Golden]), +// which snapshots what the policy stores each field as and fails, naming +// the pin, when a context changes: +// +// func TestIndividualsPolicy(t *testing.T) { +// plantest.Golden(t, source, Individuals) +// } +// // # Failing closed // // A field with annotations that no rule decides is an error when the plan diff --git a/languages/golang/stackencrypt/plan/plantest/compare.go b/languages/golang/stackencrypt/plan/plantest/compare.go new file mode 100644 index 000000000..baad1c8cb --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/compare.go @@ -0,0 +1,385 @@ +package plantest + +import ( + "fmt" + "path/filepath" + "slices" + "strings" + + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" +) + +// changes is what differs between a snapshot and the policy now, sorted +// by what each costs. +type changes struct { + // contexts lose data: a context no field writes any more. + contexts []string + // migrations need rows rewritten: a target that changed under a + // context that did not. + migrations []string + // other changes cost nothing already written: a new field, a changed + // fact, a field the policy no longer decides. + other []string +} + +// explain says what changed between the snapshot want and the policy now. +func explain(want []byte, cur snapshot, facts []plan.Fact, m plan.Message) string { + old, err := parse(want) + if err != nil { + return fmt.Sprintf("The snapshot does not parse (%v), so its changes cannot be sorted; the diff below shows them.\n\n", err) + } + return compare(old, cur, facts, m).String() +} + +func (c changes) String() string { + var b strings.Builder + section := func(title string, items []string) { + if len(items) == 0 { + return + } + b.WriteString(title) + b.WriteString("\n") + for _, it := range items { + fmt.Fprintf(&b, " - %s\n", it) + } + b.WriteString("\n") + } + section("CONTEXT CHANGES. These lose data: a context is bound into every ciphertext and query term written under it, so rows already written under the old one no longer decrypt, and their terms no longer match queries. Keep the old context by pinning the field's rule with plan.Column (and plan.Identity after a database column rename).", c.contexts) + section("TARGET CHANGES. These need a migration: rows already written keep what they were written with until they are rewritten.", c.migrations) + section("OTHER CHANGES.", c.other) + if b.Len() == 0 { + return "Nothing the policy stores changed, but the snapshot's text did: an edit by hand, or a newer plantest.\n\n" + } + return b.String() +} + +// compare sorts the differences between the snapshot old and the policy +// now (cur, built from facts by m). +func compare(old, cur snapshot, facts []plan.Fact, m plan.Message) changes { + var c changes + if old.table != cur.table { + c.contexts = append(c.contexts, fmt.Sprintf("the message's table is %s, was %s. The table is the first half of every EQL column's context; restore plan.Table(%q).", + token(cur.table), token(old.table), old.table)) + } + + oldCols, curCols := byName(old.columns), byName(cur.columns) + oldPlain, curPlain := byField(old.plaintext), byField(cur.plaintext) + oldContexts := map[string]bool{} + for _, o := range old.columns { + oldContexts[o.context] = true + } + // The entries now that an old one accounts for, and the old plaintext + // fields an entry now accounts for. + seenCol, seenPlain, seenOldPlain := map[string]bool{}, map[string]bool{}, map[string]bool{} + + for _, o := range old.columns { + if n, ok := curCols[o.name]; ok { + seenCol[n.name] = true + if n.context != o.context { + c.contexts = append(c.contexts, fmt.Sprintf("column %s: its context is %q, was %q. %s", + token(o.name), n.context, o.context, pinAdvice(m, facts, n.from, o, old.table))) + } + c.stored(o, n) + continue + } + // A database column rename: a new column under the same context. + if n, ok := only(cur.columns, func(n column) bool { + _, before := oldCols[n.name] + return !before && !seenCol[n.name] && n.context == o.context + }); ok { + seenCol[n.name] = true + c.migrations = append(c.migrations, fmt.Sprintf("column %s is now stored in column %s, under the same context: the database column is renamed with it (ALTER TABLE ... RENAME COLUMN).", + token(o.name), token(n.name))) + c.stored(o, n) + continue + } + // Decided Plaintext now: under the column's name, or the only new + // plaintext field with its facts. + if p, ok := only(cur.plaintext, func(p plain) bool { + _, before := oldPlain[p.field] + return !before && !seenPlain[p.field] && (p.field == o.name || len(o.facts) > 0 && slices.Equal(p.facts, o.facts)) + }); ok { + seenPlain[p.field] = true + c.migrations = append(c.migrations, fmt.Sprintf("column %s is now field %s, decided Plaintext. Rows already written hold ciphertexts under %q, which a migration must decrypt before the field is read as plaintext.", + token(o.name), token(p.field), o.context)) + continue + } + // The context is lost. The only new column with the same facts, + // under a context new to the snapshot, is most likely the same + // field renamed. + msg := fmt.Sprintf("column %s: no field writes its context %q any more.", token(o.name), o.context) + if n, ok := only(cur.columns, func(n column) bool { + _, before := oldCols[n.name] + return !before && !seenCol[n.name] && !oldContexts[n.context] && len(o.facts) > 0 && slices.Equal(n.facts, o.facts) + }); ok { + seenCol[n.name] = true + msg += fmt.Sprintf(" Field %s now writes column %s under %q with the same facts, so it is likely the same field renamed. %s", + fieldName(n.from), token(n.name), n.context, pinAdvice(m, facts, n.from, o, old.table)) + c.stored(o, n) + } else { + msg += fmt.Sprintf(" If its field was renamed, pin the renamed field's rule with plan.Column(%q); if it was removed, rows written under it can no longer be read through this policy.", o.name) + } + c.contexts = append(c.contexts, msg) + } + + for _, n := range cur.columns { + if seenCol[n.name] { + continue + } + if n.from != nil { + if p, ok := oldPlain[n.from.fact.Field]; ok { + if _, still := curPlain[p.field]; !still { + seenOldPlain[p.field] = true + c.migrations = append(c.migrations, fmt.Sprintf("field %s, decided Plaintext before, is now encrypted into column %s under %q. Rows already written hold its plaintext, which a migration must encrypt.", + token(p.field), token(n.name), n.context)) + continue + } + } + } + c.other = append(c.other, fmt.Sprintf("new column %s, under %q, with terms [%s].", token(n.name), n.context, termList(n.terms))) + } + + for _, o := range old.plaintext { + if n, ok := curPlain[o.field]; ok { + if !slices.Equal(o.facts, n.facts) { + c.other = append(c.other, fmt.Sprintf("plaintext field %s: its facts are [%s], were [%s].", token(o.field), factList(n.facts), factList(o.facts))) + } + continue + } + if !seenOldPlain[o.field] { + c.other = append(c.other, fmt.Sprintf("plaintext field %s is no longer decided by the policy (renamed, removed, or no longer classified).", token(o.field))) + } + } + for _, n := range cur.plaintext { + if _, before := oldPlain[n.field]; !before && !seenPlain[n.field] { + c.other = append(c.other, fmt.Sprintf("new plaintext field %s.", token(n.field))) + } + } + return c +} + +// stored compares what one column stores, the context aside. +func (c *changes) stored(o, n column) { + if !slices.Equal(o.terms, n.terms) { + c.migrations = append(c.migrations, fmt.Sprintf("column %s: its terms are [%s], were [%s]. Rows already written carry the terms they were written with until they are re-encrypted, so a query on a new term misses them.", + token(n.name), termList(n.terms), termList(o.terms))) + } + if !slices.Equal(o.facts, n.facts) { + c.other = append(c.other, fmt.Sprintf("column %s: its facts are [%s], were [%s].", token(n.name), factList(n.facts), factList(o.facts))) + } +} + +// pinAdvice says how to store the field from in the old column under the +// old context again: the old table, when the context was under it and the +// table moved; the pin that does, checked by building the plan with it; or +// why no pin can. +func pinAdvice(m plan.Message, facts []plan.Fact, from *decided, old column, oldTable string) string { + if from == nil { + return "" + } + if oldTable != string(m.Table()) && strings.HasPrefix(old.context, oldTable+"/") { + return fmt.Sprintf("Restoring plan.Table(%q) brings it back.", oldTable) + } + if pin := pin(m, facts, from, old.name, old.context); pin != "" { + return fmt.Sprintf("Pinning the rule that decides field %s with %s keeps it.", fieldName(from), pin) + } + return fmt.Sprintf("No plan.Column or plan.Identity pin on the rule that decides field %s brings it back: the context comes from the target itself (a plan.Custom context, or a change of target), so restore that.", fieldName(from)) +} + +// pin is the rule options, spelled as Go, that store the field from as +// column under context again, checked by building the plan with them; "" +// when none does. +func pin(m plan.Message, facts []plan.Fact, from *decided, column, context string) string { + if column == "" { + return "" + } + type try struct { + spelled string + opts []plan.RuleOption + } + tries := []try{{fmt.Sprintf("plan.Column(%q)", column), []plan.RuleOption{plan.Column(column)}}} + if id, ok := strings.CutPrefix(context, string(m.Table())+"/"); ok && id != "" { + tries = append(tries, + try{fmt.Sprintf("plan.Identity(%q)", id), []plan.RuleOption{plan.Identity(id)}}, + try{fmt.Sprintf("plan.Column(%q), plan.Identity(%q)", column, id), []plan.RuleOption{plan.Column(column), plan.Identity(id)}}, + ) + } + for _, t := range tries { + rule := plan.When(plan.Field(from.fact.Field), from.decision, t.opts...) + p, err := plan.ForMessage(m.Msg(), m.Table(), rule.OrElse(m.Decide)).Build(facts) + if err != nil { + continue + } + for _, fp := range p.Fields() { + if fp.Field == goField(from.fact) && fp.Name == column && fp.Context == context { + return t.spelled + } + } + } + return "" +} + +// only is the one element of s that keep accepts, if exactly one does. +func only[T any](s []T, keep func(T) bool) (T, bool) { + var found T + n := 0 + for _, v := range s { + if keep(v) { + found = v + n++ + } + } + if n != 1 { + var zero T + return zero, false + } + return found, true +} + +func byName(cs []column) map[string]column { + out := make(map[string]column, len(cs)) + for _, c := range cs { + out[c.name] = c + } + return out +} + +func byField(ps []plain) map[string]plain { + out := make(map[string]plain, len(ps)) + for _, p := range ps { + out[p.field] = p + } + return out +} + +// fieldName names a field the way a rule matches it, and the Go field it +// binds to when that is spelled differently. +func fieldName(d *decided) string { + if d == nil { + return "?" + } + name := token(d.fact.Field) + if g := goField(d.fact); g != d.fact.Field { + name += " (" + g + ")" + } + return name +} + +func factList(fs []fact) string { + if len(fs) == 0 { + return "none" + } + parts := make([]string, len(fs)) + for i, f := range fs { + parts[i] = token(f.key) + "=" + token(f.value) + } + return strings.Join(parts, " ") +} + +// diff is a unified diff of the snapshot want and the policy now, with +// two lines of context. +func diff(path string, want, got []byte) string { + es := edits(lines(want), lines(got)) + const context = 2 + keep := make([]bool, len(es)) + for i, e := range es { + if e.op == ' ' { + continue + } + for j := max(0, i-context); j <= min(len(es)-1, i+context); j++ { + keep[j] = true + } + } + var b strings.Builder + fmt.Fprintf(&b, "--- %s\n+++ the policy now\n", filepath.ToSlash(path)) + for i := 0; i < len(es); { + if !keep[i] { + i++ + continue + } + end := i + for end < len(es) && keep[end] { + end++ + } + var olds, news int + for _, e := range es[i:end] { + if e.op != '+' { + olds++ + } + if e.op != '-' { + news++ + } + } + fmt.Fprintf(&b, "@@ -%d,%d +%d,%d @@\n", es[i].a+1, olds, es[i].b+1, news) + for _, e := range es[i:end] { + b.WriteByte(e.op) + b.WriteString(e.line) + b.WriteByte('\n') + } + i = end + } + return b.String() +} + +// edit is one line of a diff: kept (' '), removed ('-') or added ('+'), +// with the index each side has reached. +type edit struct { + op byte + line string + a, b int +} + +// edits is a shortest edit script from a to b, by longest common +// subsequence. Snapshots are small; past a few million cells it gives up +// on alignment and replaces a with b wholesale. +func edits(a, b []string) []edit { + var out []edit + if len(a)*len(b) > 1<<22 { + for i, l := range a { + out = append(out, edit{'-', l, i, 0}) + } + for j, l := range b { + out = append(out, edit{'+', l, len(a), j}) + } + return out + } + lcs := make([][]int32, len(a)+1) + for i := range lcs { + lcs[i] = make([]int32, len(b)+1) + } + for i := len(a) - 1; i >= 0; i-- { + for j := len(b) - 1; j >= 0; j-- { + if a[i] == b[j] { + lcs[i][j] = lcs[i+1][j+1] + 1 + } else { + lcs[i][j] = max(lcs[i+1][j], lcs[i][j+1]) + } + } + } + i, j := 0, 0 + for i < len(a) || j < len(b) { + switch { + case i < len(a) && j < len(b) && a[i] == b[j]: + out = append(out, edit{' ', a[i], i, j}) + i++ + j++ + case j == len(b) || i < len(a) && lcs[i+1][j] >= lcs[i][j+1]: + out = append(out, edit{'-', a[i], i, j}) + i++ + default: + out = append(out, edit{'+', b[j], i, j}) + j++ + } + } + return out +} + +// lines splits text into its lines, without the final newline's empty +// one. +func lines(text []byte) []string { + s := strings.TrimSuffix(string(text), "\n") + if s == "" { + return nil + } + return strings.Split(s, "\n") +} diff --git a/languages/golang/stackencrypt/plan/plantest/golden_test.go b/languages/golang/stackencrypt/plan/plantest/golden_test.go new file mode 100644 index 000000000..4d71d86b4 --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/golden_test.go @@ -0,0 +1,47 @@ +package plantest_test + +import ( + "testing" + + "github.com/cipherstash/stack/languages/golang/internal/factstest" + se "github.com/cipherstash/stack/languages/golang/stackencrypt" + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan/plantest" +) + +var category = plan.Key("fides.data_categories") + +type individual struct { + ID int64 + Email string `facts:"fides.data_categories=user.contact.email"` + Name string `facts:"fides.data_categories=user.name"` + MedicareNo string `facts:"fides.data_categories=user.government_id"` + Notes []byte `facts:"fides.data_categories=user.content"` + Country string `facts:"fides.data_categories=system.operations"` +} + +// individuals has a field whose database column was renamed (stored in +// medicare_num, under its first identity), a Custom target, and a field +// left in plaintext. +var individuals = plan.ForMessage(&individual{}, plan.Table("individuals"), plan.FirstOf( + plan.When(plan.Field("medicare_no"), plan.Encrypt(plan.EQL(se.Equality)), + plan.Column("medicare_num"), plan.Identity("medicare_number")), + plan.When(category.Under("user.content"), plan.Encrypt(plan.Custom("individuals-notes/v1"))), + plan.When(category.Under("user.contact.email"), plan.Encrypt(plan.EQL(se.Equality, se.Match))), + plan.When(category.Under("user"), plan.Encrypt(plan.EQL(se.Ore))), + plan.When(category.Under("system"), plan.Plaintext()), +)) + +type audit struct { + ID int64 + Kind string `facts:"fides.data_categories=system.operations"` +} + +// audits encrypts nothing: its snapshot is the fields it leaves plaintext. +var audits = plan.ForMessage(audit{}, plan.Table("audits"), plan.When(category.Under("system"), plan.Plaintext())) + +// Each message's snapshot is checked in at testdata/TestPolicies/.golden. +func TestPolicies(t *testing.T) { + t.Run("individuals", func(t *testing.T) { plantest.Golden(t, factstest.StructTags, individuals) }) + t.Run("audits", func(t *testing.T) { plantest.Golden(t, factstest.StructTags, audits) }) +} diff --git a/languages/golang/stackencrypt/plan/plantest/plantest.go b/languages/golang/stackencrypt/plan/plantest/plantest.go new file mode 100644 index 000000000..e673717c6 --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/plantest.go @@ -0,0 +1,185 @@ +// Package plantest checks in what a [plan.Policy] stores each field as, so +// a change to a field's encryption context fails a test instead of the +// data. +// +// A field's context is bound into every ciphertext and query term written +// under it. A policy derives contexts from names, so an ordinary rename — +// a proto field, a Go struct field, a table — can change one with no error +// on the write path: rows already written stop decrypting, and their terms +// stop matching queries. [Golden] makes that a test failure: +// +// func TestIndividualsPolicy(t *testing.T) { +// plantest.Golden(t, source, policy.Individuals) +// } +// +// The first run with -update writes the snapshot to +// testdata/TestIndividualsPolicy.golden; check it in. From then on Golden +// builds the plan the way [plan.MustPlanFor] does at startup, and fails +// when the plan no longer matches the snapshot, naming each change and +// calling out a changed context apart from the rest: a changed context is +// data loss, while a changed target (its index terms, or encrypting a +// field that was plaintext) is a migration. When a rename changed a +// context, the failure names the [plan.Column] (or [plan.Identity]) pin +// that keeps it, checked by building the plan with that pin. +// +// # The snapshot +// +// The snapshot records what a policy stores, not what the schema calls it: +// per message its [plan.Table]; per encrypted field its column (the record +// key), its context, its index terms and its facts; per field decided +// [plan.Plaintext], its name and its facts. Fields with no facts that no +// rule names are not the policy's concern and are left out. Encrypted +// fields are named by column, so a schema rename that the policy pins +// leaves the snapshot byte-for-byte unchanged, which is how a reviewer +// tells a safe rename from one that loses data. +// +// The text is line-oriented and deterministic across runs and platforms: +// fields are sorted, facts are sorted, line endings are "\n" (a checkout +// that converted them to "\r\n" still compares equal), and any value that +// is not a plain identifier is quoted as a Go string literal. It is meant +// to be read by people as well as compared: the list of classified fields, +// what protects each, and the identifier it is bound to. +// +// # The -update flag +// +// plantest registers the conventional -update test flag, as a bool on the +// default flag set, when no package it imports has done so already. A test +// package that defines its own -update panics with "flag redefined"; +// read plantest's instead, with flag.Lookup("update"). +package plantest + +import ( + "errors" + "flag" + "fmt" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" +) + +func init() { + if flag.Lookup("update") == nil { + flag.Bool("update", false, "rewrite plantest golden snapshots (testdata/*.golden) from the policy") + } +} + +// updating reports whether the test binary was run with -update. +func updating() bool { + f := flag.Lookup("update") + if f == nil { + return false + } + g, ok := f.Value.(flag.Getter) + if !ok { + return false + } + on, ok := g.Get().(bool) + return ok && on +} + +// Golden builds m's plan over src's facts, as [plan.PlanFor] does, and +// checks what it stores each field as against the snapshot checked in at +// testdata/.golden (a subtest's name is a path below +// testdata). With -update it writes the snapshot instead, logging what +// changed. +// +// The test fails when the policy does not build (a classified field no +// rule decides, a [plan.Fail], a plan that does not bind to the message), +// when there is no snapshot yet, and when the plan differs from it. A +// message the policy encrypts nothing of is not a failure: its snapshot +// lists the fields it decided Plaintext. +// +// A [plan.Source] is expected to be pure, as a policy is: Golden reads the +// facts more than once. +func Golden(t testing.TB, src plan.Source, m plan.Message) { + t.Helper() + logs, err := check(goldenPath(t.Name()), src, m, updating(), rerun(t.Name())) + if logs != "" { + t.Log(logs) + } + if err != nil { + t.Error(err) + } +} + +// goldenPath is the snapshot file for a test: testdata/.golden, a +// subtest's name a path below it. A character a file system could refuse +// is spelled '_'. +func goldenPath(name string) string { + parts := strings.Split(name, "/") + for i, p := range parts { + parts[i] = strings.Map(func(r rune) rune { + if r < 0x80 && (r == '-' || r == '_' || r == '.' || r == '+' || + 'a' <= r && r <= 'z' || 'A' <= r && r <= 'Z' || '0' <= r && r <= '9') { + return r + } + return '_' + }, p) + if strings.Trim(parts[i], ".") == "" { + parts[i] = strings.Repeat("_", len(parts[i])+1) + } + } + return filepath.Join(append([]string{"testdata"}, parts...)...) + ".golden" +} + +// rerun is the command that runs just this test with -update. +func rerun(name string) string { + parts := strings.Split(name, "/") + for i, p := range parts { + parts[i] = "^" + regexp.QuoteMeta(p) + "$" + } + return fmt.Sprintf("go test -run '%s' -update", strings.Join(parts, "/")) +} + +// check is Golden without the testing.TB: it compares (or, updating, +// writes) the snapshot at path, and returns what to log and what failed. +func check(path string, src plan.Source, m plan.Message, update bool, rerun string) (string, error) { + cur, facts, err := take(src, m) + if err != nil { + return "", fmt.Errorf("plantest: the policy does not build, so there is nothing to snapshot: %w", err) + } + got := cur.render() + path = filepath.Clean(path) + want, readErr := os.ReadFile(path) + if readErr != nil && !errors.Is(readErr, fs.ErrNotExist) { + return "", fmt.Errorf("plantest: %w", readErr) + } + same := readErr == nil && string(normalize(want)) == string(got) + + if update { + if same { + return "", nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + return "", fmt.Errorf("plantest: %w", err) + } + // A snapshot is checked in, not secret; git records no mode but + // the executable bit. + if err := os.WriteFile(path, got, 0o644); err != nil { //nolint:gosec // see above + return "", fmt.Errorf("plantest: %w", err) + } + if readErr != nil { + return fmt.Sprintf("plantest: wrote %s; review it and check it in", path), nil + } + return fmt.Sprintf("plantest: updated %s, recording:\n\n%s", path, strings.TrimRight(explain(want, cur, facts, m), "\n")), nil + } + + if readErr != nil { + return "", fmt.Errorf("plantest: no snapshot at %s. Write it with\n\n\t%s\n\nthen review it and check it in", path, rerun) + } + if same { + return "", nil + } + return "", fmt.Errorf("plantest: %s no longer matches the policy.\n\n%sIf every change is intended (no row has been written under a changed context yet, or a migration ships with it), record them with\n\n\t%s\n\n%s", + path, explain(want, cur, facts, m), rerun, diff(path, normalize(want), got)) +} + +// normalize undoes a checkout's "\r\n" line endings. +func normalize(b []byte) []byte { + return []byte(strings.ReplaceAll(string(b), "\r\n", "\n")) +} diff --git a/languages/golang/stackencrypt/plan/plantest/plantest_internal_test.go b/languages/golang/stackencrypt/plan/plantest/plantest_internal_test.go new file mode 100644 index 000000000..20ba432a1 --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/plantest_internal_test.go @@ -0,0 +1,471 @@ +package plantest + +import ( + "bytes" + "errors" + "flag" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/cipherstash/stack/languages/golang/internal/factstest" + se "github.com/cipherstash/stack/languages/golang/stackencrypt" + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" +) + +var category = plan.Key("fides.data_categories") + +var base = plan.FirstOf( + plan.When(category.Under("user.government_id"), plan.Encrypt(plan.EQL(se.Equality))), + plan.When(category.Under("user.contact.email"), plan.Encrypt(plan.EQL(se.Equality, se.Match))), + plan.When(category.Under("user"), plan.Encrypt(plan.EQL())), + plan.When(category.Under("system"), plan.Plaintext()), +) + +func classified(values ...string) []plan.Annotation { + return []plan.Annotation{{Key: "fides.data_categories", Values: values}} +} + +// proto is a protobuf-shaped source: the facts are given, and a field +// keeps its number through a rename. +func proto(facts ...plan.Fact) plan.Source { + for i := range facts { + facts[i].Message = "acme.v1.Individual" + } + return plan.SourceFunc(func(any) ([]plan.Fact, error) { return facts, nil }) +} + +// individualV1 is the schema before a rename. +func individualV1() plan.Source { + return proto( + plan.Fact{Field: "id", GoField: "Id", Number: 1, Kind: "int64"}, + plan.Fact{Field: "email", GoField: "Email", Number: 2, Kind: "string", Annotations: classified("user.contact.email")}, + plan.Fact{Field: "medicare_number", GoField: "MedicareNumber", Number: 3, Kind: "string", Annotations: classified("user.government_id")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Kind: "string", Annotations: classified("system.operations")}, + ) +} + +// individualV2 is individualV1 with field 3 renamed. +func individualV2() plan.Source { + return proto( + plan.Fact{Field: "id", GoField: "Id", Number: 1, Kind: "int64"}, + plan.Fact{Field: "email", GoField: "Email", Number: 2, Kind: "string", Annotations: classified("user.contact.email")}, + plan.Fact{Field: "medicare_no", GoField: "MedicareNo", Number: 3, Kind: "string", Annotations: classified("user.government_id")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Kind: "string", Annotations: classified("system.operations")}, + ) +} + +// record writes the snapshot of m over src to a fresh path and returns +// the path and what it wrote. +func record(t *testing.T, src plan.Source, m plan.Message) (string, []byte) { + t.Helper() + path := filepath.Join(t.TempDir(), "testdata", "TestPolicy.golden") + logs, err := check(path, src, m, true, "RERUN") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(logs, "wrote "+path) { + t.Errorf("logs = %q, want the path written", logs) + } + written, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return path, written +} + +func mustContain(t *testing.T, err error, wants ...string) { + t.Helper() + if err == nil { + t.Fatalf("passed, want a failure saying %q", wants) + } + for _, want := range wants { + if !strings.Contains(err.Error(), want) { + t.Errorf("failure does not say %q:\n%s", want, err) + } + } +} + +// The acceptance case: a renamed proto field with no pin fails with a +// context change, naming the pin; with the pin it passes, and the +// snapshot is the one checked in, byte for byte. +func TestRenameWithoutAPinIsAContextChange(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, before := record(t, individualV1(), m) + + _, err := check(path, individualV2(), m, false, "RERUN") + mustContain(t, err, + "CONTEXT CHANGES", + `column medicare_number: no field writes its context "individuals/medicare_number" any more.`, + `Field medicare_no (MedicareNo) now writes column medicare_no under "individuals/medicare_no"`, + `Pinning the rule that decides field medicare_no (MedicareNo) with plan.Column("medicare_number") keeps it.`, + "RERUN", + "- context individuals/medicare_number", + "+ context individuals/medicare_no", + ) + // A context change is data loss, not a migration or a new column. + if strings.Contains(err.Error(), "TARGET CHANGES") || strings.Contains(err.Error(), "OTHER CHANGES") { + t.Errorf("a rename is reported as more than a context change:\n%s", err) + } + + pinned := plan.ForMessage(nil, "individuals", plan.FirstOf( + plan.When(plan.Field("medicare_no"), plan.Encrypt(plan.EQL(se.Equality)), plan.Column("medicare_number")), + ).OrElse(base)) + if _, err := check(path, individualV2(), pinned, false, "RERUN"); err != nil { + t.Fatalf("the pinned rename fails: %v", err) + } + after, _, err := take(individualV2(), pinned) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(after.render(), before) { + t.Fatalf("the pinned rename changed the snapshot:\n%s\nwas\n%s", after.render(), before) + } +} + +// The same holds for a Go struct field renamed, where a fact source has +// no field number and the plan binds to the struct. +func TestStructFieldRename(t *testing.T) { + type v1 struct { + ID int64 + MedicareNumber string `facts:"fides.data_categories=user.government_id"` + } + type v2 struct { + ID int64 + MedicareNo string `facts:"fides.data_categories=user.government_id"` + } + path, _ := record(t, factstest.StructTags, plan.ForMessage(v1{}, "individuals", base)) + _, err := check(path, factstest.StructTags, plan.ForMessage(v2{}, "individuals", base), false, "RERUN") + mustContain(t, err, "CONTEXT CHANGES", `plan.Column("medicare_number")`) + pinned := plan.ForMessage(v2{}, "individuals", plan.FirstOf( + plan.When(plan.Field("medicare_no"), plan.Encrypt(plan.EQL(se.Equality)), plan.Column("medicare_number")), + ).OrElse(base)) + if _, err := check(path, factstest.StructTags, pinned, false, "RERUN"); err != nil { + t.Fatalf("the pinned rename fails: %v", err) + } +} + +// Each kind of change lands in its own section, with the advice that +// fits it. +func TestChangesAreSortedByWhatTheyCost(t *testing.T) { + gov := plan.Encrypt(plan.EQL(se.Equality)) + for name, tc := range map[string]struct { + before, after plan.Message + src plan.Source // individualV1 when nil + section string // the only section reported; "" when it does not build + says []string + }{ + "identity pin dropped": { + before: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), gov, plan.Identity("medicare_no"))).OrElse(base)), + after: plan.ForMessage(nil, "individuals", base), + section: "CONTEXT CHANGES", + says: []string{`column medicare_number: its context is "individuals/medicare_number", was "individuals/medicare_no".`, `with plan.Identity("medicare_no") keeps it.`}, + }, + "table changed": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "people", base), + section: "CONTEXT CHANGES", + says: []string{`the message's table is people, was individuals.`, `restore plan.Table("individuals")`, `its context is "people/email", was "individuals/email". Restoring plan.Table("individuals") brings it back.`}, + }, + "custom context changed": { + before: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), plan.Encrypt(plan.Custom("gov/v1")))).OrElse(base)), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), plan.Encrypt(plan.Custom("gov/v2")))).OrElse(base)), + section: "CONTEXT CHANGES", + says: []string{`its context is "gov/v2", was "gov/v1". No plan.Column or plan.Identity pin`, "a plan.Custom context"}, + }, + "encrypted now plaintext": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), plan.Plaintext())).OrElse(base)), + section: "TARGET CHANGES", + says: []string{"column medicare_number is now field medicare_number, decided Plaintext.", `ciphertexts under "individuals/medicare_number"`}, + }, + "terms changed": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("email"), plan.Encrypt(plan.EQL(se.Equality)))).OrElse(base)), + section: "TARGET CHANGES", + says: []string{"column email: its terms are [eq], were [eq match]."}, + }, + "plaintext now encrypted": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("country"), plan.Encrypt(plan.EQL()))).OrElse(base)), + section: "TARGET CHANGES", + says: []string{`field country, decided Plaintext before, is now encrypted into column country under "individuals/country".`}, + }, + "database column renamed": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("medicare_number"), gov, plan.Column("medicare_num"), plan.Identity("medicare_number"))).OrElse(base)), + section: "TARGET CHANGES", + says: []string{"column medicare_number is now stored in column medicare_num, under the same context"}, + }, + "field newly classified": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("id"), plan.Encrypt(plan.EQL(se.Ore)))).OrElse(base)), + section: "OTHER CHANGES", + says: []string{`new column id, under "individuals/id", with terms [ore].`}, + }, + "facts changed": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", base), + src: proto( + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: classified("user.contact.email", "user.contact.email.work")}, + plan.Fact{Field: "medicare_number", GoField: "MedicareNumber", Number: 3, Annotations: classified("user.government_id")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Annotations: classified("system.operations", "system.location")}, + ), + section: "OTHER CHANGES", + says: []string{ + "column email: its facts are [fides.data_categories=user.contact.email fides.data_categories=user.contact.email.work], were [fides.data_categories=user.contact.email].", + "plaintext field country: its facts are", + }, + }, + "plaintext field gone": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", base), + src: proto( + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: classified("user.contact.email")}, + plan.Fact{Field: "medicare_number", GoField: "MedicareNumber", Number: 3, Annotations: classified("user.government_id")}, + ), + section: "OTHER CHANGES", + says: []string{"plaintext field country is no longer decided by the policy"}, + }, + "refused": { + before: plan.ForMessage(nil, "individuals", base), + after: plan.ForMessage(nil, "individuals", plan.FirstOf(plan.When(plan.Field("country"), plan.Fail("never stored"))).OrElse(base)), + says: []string{"the policy does not build", "never stored"}, + }, + } { + t.Run(name, func(t *testing.T) { + path, _ := record(t, individualV1(), tc.before) + src := tc.src + if src == nil { + src = individualV1() + } + _, err := check(path, src, tc.after, false, "RERUN") + mustContain(t, err, tc.says...) + if tc.section == "" { + return + } + for _, s := range []string{"CONTEXT CHANGES", "TARGET CHANGES", "OTHER CHANGES"} { + if got := strings.Contains(err.Error(), s); got != (s == tc.section) { + t.Errorf("reports %s = %v, want only %s:\n%s", s, got, tc.section, err) + } + } + }) + } +} + +// A field removed from the schema without a trace is a lost context: the +// failure says what to do either way. +func TestALostContextWithNoCandidate(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, _ := record(t, individualV1(), m) + gone := proto( + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: classified("user.contact.email")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Annotations: classified("system.operations")}, + ) + _, err := check(path, gone, m, false, "RERUN") + mustContain(t, err, "CONTEXT CHANGES", `column medicare_number: no field writes its context "individuals/medicare_number" any more. If its field was renamed, pin the renamed field's rule with plan.Column("medicare_number")`) +} + +// Two new columns with the facts of the one that disappeared: no guess, +// and the generic advice. +func TestAmbiguousRenameIsNotGuessed(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, _ := record(t, individualV1(), m) + split := proto( + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: classified("user.contact.email")}, + plan.Fact{Field: "medicare_a", GoField: "MedicareA", Number: 5, Annotations: classified("user.government_id")}, + plan.Fact{Field: "medicare_b", GoField: "MedicareB", Number: 6, Annotations: classified("user.government_id")}, + plan.Fact{Field: "country", GoField: "Country", Number: 4, Annotations: classified("system.operations")}, + ) + _, err := check(path, split, m, false, "RERUN") + mustContain(t, err, "If its field was renamed", "new column medicare_a", "new column medicare_b") + if strings.Contains(err.Error(), "likely the same field renamed") { + t.Errorf("guessed a rename between two candidates:\n%s", err) + } +} + +// The snapshot is the same whatever order the source gives fields and +// facts in, and from run to run. +func TestSnapshotIsDeterministic(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + a, _, err := take(individualV1(), m) + if err != nil { + t.Fatal(err) + } + shuffled := proto( + plan.Fact{Field: "country", GoField: "Country", Number: 4, Annotations: classified("system.operations")}, + plan.Fact{Field: "medicare_number", GoField: "MedicareNumber", Number: 3, Annotations: classified("user.government_id")}, + plan.Fact{Field: "email", GoField: "Email", Number: 2, Annotations: []plan.Annotation{ + {Key: "fides.data_categories", Values: []string{"user.contact.email"}}, + {Key: "fides.data_categories", Values: []string{"user.contact.email"}}, + }}, + plan.Fact{Field: "id", GoField: "Id", Number: 1}, + ) + b, _, err := take(shuffled, m) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(a.render(), b.render()) { + t.Fatalf("field or fact order changed the snapshot:\n%s\nvs\n%s", a.render(), b.render()) + } + want := header + ` +table individuals + +column email + context individuals/email + terms eq match + fact fides.data_categories user.contact.email + +column medicare_number + context individuals/medicare_number + terms eq + fact fides.data_categories user.government_id + +plaintext country + fact fides.data_categories system.operations +` + if got := string(a.render()); got != want { + t.Fatalf("snapshot =\n%s\nwant\n%s", got, want) + } +} + +// Whatever a name, a context or a fact holds, the snapshot reads back as +// what was written. +func TestSnapshotRoundTrips(t *testing.T) { + odd := []string{"", "with space", `"quoted"`, "new\nline", "tab\there", "naïve", "#hash", "none", "a=b,c"} + s := snapshot{table: "odd table"} + for i, v := range odd { + s.columns = append(s.columns, column{name: v + string(rune('a'+i)), context: v, terms: []string{"eq", "ore"}, facts: []fact{{v, v}, {"k", v}}}) + s.plaintext = append(s.plaintext, plain{field: v + string(rune('a'+i)), facts: []fact{{v, "x"}}}) + } + s.columns = append(s.columns, column{name: "bare", context: "t/bare"}) + for i := range s.columns { + sortFacts(s.columns[i].facts) + } + s.sort() + text := s.render() + back, err := parse(text) + if err != nil { + t.Fatalf("%v\n%s", err, text) + } + if again := back.render(); !bytes.Equal(again, text) { + t.Fatalf("round trip changed the snapshot:\n%s\nwas\n%s", again, text) + } + if !strings.Contains(string(text), " terms none\n") { + t.Errorf("an unindexed column does not say so:\n%s", text) + } +} + +func TestMissingSnapshot(t *testing.T) { + path := filepath.Join(t.TempDir(), "testdata", "TestPolicy.golden") + _, err := check(path, individualV1(), plan.ForMessage(nil, "individuals", base), false, "go test -run '^TestPolicy$' -update") + mustContain(t, err, "no snapshot at "+path, "go test -run '^TestPolicy$' -update") +} + +func TestUpdateRewritesAndSaysWhatItRecorded(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, _ := record(t, individualV1(), m) + logs, err := check(path, individualV2(), m, true, "RERUN") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(logs, "updated "+path) || !strings.Contains(logs, "CONTEXT CHANGES") { + t.Errorf("logs = %q, want the update and the context change it recorded", logs) + } + if _, err := check(path, individualV2(), m, false, "RERUN"); err != nil { + t.Fatalf("after -update: %v", err) + } + // Updating an unchanged snapshot is silent and leaves it alone. + if logs, err := check(path, individualV2(), m, true, "RERUN"); err != nil || logs != "" { + t.Errorf("no-op update: logs %q, err %v", logs, err) + } +} + +// A checkout that converted line endings still matches. +func TestCRLFCheckoutMatches(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, written := record(t, individualV1(), m) + if err := os.WriteFile(path, bytes.ReplaceAll(written, []byte("\n"), []byte("\r\n")), 0o600); err != nil { + t.Fatal(err) + } + if _, err := check(path, individualV1(), m, false, "RERUN"); err != nil { + t.Fatal(err) + } +} + +func TestTextOnlyAndUnreadableSnapshots(t *testing.T) { + m := plan.ForMessage(nil, "individuals", base) + path, written := record(t, individualV1(), m) + + // Same content, different text: still a failure, said as such. + edited := bytes.Replace(written, []byte("table individuals"), []byte(`table "individuals"`), 1) + if err := os.WriteFile(path, edited, 0o600); err != nil { + t.Fatal(err) + } + _, err := check(path, individualV1(), m, false, "RERUN") + mustContain(t, err, "Nothing the policy stores changed", `-table "individuals"`, "+table individuals") + + if err := os.WriteFile(path, append(written, []byte("garbage line\n")...), 0o600); err != nil { + t.Fatal(err) + } + _, err = check(path, individualV1(), m, false, "RERUN") + mustContain(t, err, "does not parse", `unexpected "garbage line"`, "+++ the policy now") +} + +// A policy that does not build fails the test with the build's error. +func TestPolicyThatDoesNotBuild(t *testing.T) { + narrow := plan.ForMessage(nil, "individuals", plan.When(category.Under("user.contact"), plan.Encrypt(plan.EQL()))) + _, err := check(filepath.Join(t.TempDir(), "x.golden"), individualV1(), narrow, true, "RERUN") + if !errors.Is(err, plan.ErrUnmatched) { + t.Fatalf("err = %v, want ErrUnmatched", err) + } + mustContain(t, err, "the policy does not build", "medicare_number") + if _, err := check("x.golden", nil, narrow, true, "RERUN"); err == nil || !strings.Contains(err.Error(), "needs a Source") { + t.Errorf("nil source: %v", err) + } +} + +// A message the policy encrypts nothing of has a snapshot: the fields it +// decided Plaintext. +func TestNothingEncrypted(t *testing.T) { + m := plan.ForMessage(nil, "individuals", plan.When(category.Present(), plan.Plaintext())) + _, written := record(t, individualV1(), m) + if strings.Contains(string(written), "column ") || !strings.Contains(string(written), "plaintext medicare_number") { + t.Fatalf("snapshot:\n%s", written) + } +} + +func TestGoldenPathAndRerun(t *testing.T) { + for name, want := range map[string]string{ + "TestPolicy": filepath.Join("testdata", "TestPolicy.golden"), + "TestPolicy/individuals": filepath.Join("testdata", "TestPolicy", "individuals.golden"), + "TestPolicy/a:b*c?": filepath.Join("testdata", "TestPolicy", "a_b_c_.golden"), + "TestPolicy/..": filepath.Join("testdata", "TestPolicy", "___.golden"), + "TestPolicy/v1.2+build_name-x": filepath.Join("testdata", "TestPolicy", "v1.2+build_name-x.golden"), + } { + if got := goldenPath(name); got != want { + t.Errorf("goldenPath(%q) = %q, want %q", name, got, want) + } + } + if got, want := rerun("TestPolicy/a.b"), `go test -run '^TestPolicy$/^a\.b$' -update`; got != want { + t.Errorf("rerun = %s, want %s", got, want) + } +} + +func TestUpdateFlag(t *testing.T) { + f := flag.Lookup("update") + if f == nil { + t.Fatal("-update is not registered") + } + if updating() { + t.Skip("run with -update") + } + if err := f.Value.Set("true"); err != nil { + t.Fatal(err) + } + defer func() { _ = f.Value.Set("false") }() + if !updating() { + t.Error("updating() does not follow -update") + } +} diff --git a/languages/golang/stackencrypt/plan/plantest/snapshot.go b/languages/golang/stackencrypt/plan/plantest/snapshot.go new file mode 100644 index 000000000..1cc8a6135 --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/snapshot.go @@ -0,0 +1,282 @@ +package plantest + +import ( + "bytes" + "errors" + "fmt" + "slices" + "strconv" + "strings" + + "github.com/cipherstash/stack/languages/golang/stackencrypt" + "github.com/cipherstash/stack/languages/golang/stackencrypt/plan" +) + +// header opens every snapshot. One sentence per line: the file is read in +// diffs, where a rewrapped paragraph would read as a change. +const header = `# Written by plantest.Golden: what the policy stores each field it decides as. +# Regenerate it with go test -update; do not edit it by hand. +# A column's context is bound into every ciphertext and query term written under it, so once a row is written it must never change. +` + +// snapshot is what a message's policy stores each field it decides as: +// the persistent half of a plan, which a rename the policy pins leaves +// alone. +type snapshot struct { + table string + columns []column // encrypted fields, by column + plaintext []plain // fields decided Plaintext, by field +} + +// column is one encrypted field, named by its record key. +type column struct { + name string + context string + terms []string + facts []fact + // from is the field it is decided from now; nil when read from a file. + from *decided +} + +// plain is one field decided Plaintext, named by its schema name. +type plain struct { + field string + facts []fact + from *decided +} + +// decided is a field of the message as the policy sees it now. +type decided struct { + fact plan.Fact + decision plan.Decision +} + +// fact is one annotation value. +type fact struct{ key, value string } + +// take builds m's plan from src, as plan.PlanFor does at startup, and +// snapshots it. It returns the facts as well, for the hints a comparison +// builds. +func take(src plan.Source, m plan.Message) (snapshot, []plan.Fact, error) { + if src == nil { + return snapshot{}, nil, errors.New("plantest: Golden needs a Source") + } + p, err := plan.PlanFor(src, m) + if err != nil && !errors.Is(err, plan.ErrNothingEncrypted) { + return snapshot{}, nil, err + } + facts, err := src.Facts(m.Msg()) + if err != nil { + return snapshot{}, nil, err + } + planned := map[string]stackencrypt.FieldPlan{} + for _, fp := range p.Fields() { + planned[fp.Field] = fp + } + s := snapshot{table: string(m.Table())} + seen := map[string]bool{} + for _, f := range facts { + d, ok := m.Decide(f) + if !ok { + // Unclassified and unnamed: PlanFor has already refused a + // classified field no rule decides. + continue + } + if seen[f.Field] { + return snapshot{}, nil, fmt.Errorf("plantest: the source gives field %q twice", f.Field) + } + seen[f.Field] = true + from := &decided{fact: f, decision: d} + if _, encrypted := d.Target(); !encrypted { + s.plaintext = append(s.plaintext, plain{field: f.Field, facts: factsOf(f), from: from}) + continue + } + fp, ok := planned[goField(f)] + if !ok { + return snapshot{}, nil, fmt.Errorf("plantest: %s is decided %v but is not in the plan", f, d) + } + terms := make([]string, len(fp.Terms)) + for i, k := range fp.Terms { + terms[i] = k.String() + } + s.columns = append(s.columns, column{name: fp.Name, context: fp.Context, terms: terms, facts: factsOf(f), from: from}) + } + s.sort() + return s, facts, nil +} + +// goField is the Go field a fact binds to, as the plan package spells it. +func goField(f plan.Fact) string { + if f.GoField != "" { + return f.GoField + } + return f.Field +} + +// factsOf is a field's annotations as sorted, distinct (key, value) pairs: +// their order in the schema means nothing to a policy. +func factsOf(f plan.Fact) []fact { + var out []fact + for _, a := range f.Annotations { + for _, v := range a.Values { + out = append(out, fact{a.Key, v}) + } + } + sortFacts(out) + return slices.Compact(out) +} + +func sortFacts(fs []fact) { + slices.SortFunc(fs, func(a, b fact) int { + if c := strings.Compare(a.key, b.key); c != 0 { + return c + } + return strings.Compare(a.value, b.value) + }) +} + +func (s *snapshot) sort() { + slices.SortFunc(s.columns, func(a, b column) int { return strings.Compare(a.name, b.name) }) + slices.SortFunc(s.plaintext, func(a, b plain) int { return strings.Compare(a.field, b.field) }) +} + +// render spells the snapshot. parse reads it back. +func (s snapshot) render() []byte { + var b bytes.Buffer + b.WriteString(header) + fmt.Fprintf(&b, "\ntable %s\n", token(s.table)) + for _, c := range s.columns { + fmt.Fprintf(&b, "\ncolumn %s\n", token(c.name)) + fmt.Fprintf(&b, " context %s\n", token(c.context)) + fmt.Fprintf(&b, " terms %s\n", termList(c.terms)) + writeFacts(&b, c.facts) + } + for _, p := range s.plaintext { + fmt.Fprintf(&b, "\nplaintext %s\n", token(p.field)) + writeFacts(&b, p.facts) + } + return b.Bytes() +} + +// termList spells a column's terms, "none" for none: an unindexed column +// is a decision too, and says nothing about the value but its ciphertext. +func termList(terms []string) string { + if len(terms) == 0 { + return "none" + } + return strings.Join(terms, " ") +} + +func writeFacts(b *bytes.Buffer, facts []fact) { + for _, f := range facts { + fmt.Fprintf(b, " fact %s %s\n", token(f.key), token(f.value)) + } +} + +// token spells a value: bare when it is a plain identifier, a Go string +// literal otherwise, so a space, a quote or a newline in a name or a +// context cannot change how the line reads. +func token(s string) string { + if s == "" { + return strconv.Quote(s) + } + for i := 0; i < len(s); i++ { + if !bare(s[i]) { + return strconv.Quote(s) + } + } + return s +} + +// bare reports whether c may appear in an unquoted value. +func bare(c byte) bool { + return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' || + c == '_' || c == '.' || c == '/' || c == '-' || c == ':' || c == '@' || c == '+' +} + +// parse reads a snapshot render wrote. It is strict about structure and +// lenient about nothing else: any byte difference fails the comparison +// anyway, and parse only lets the failure say what changed. +func parse(data []byte) (snapshot, error) { + var s snapshot + var table bool + // The entry the indented lines below belong to: a column (>= 0), a + // plaintext field (plainAt >= 0), or neither. + colAt, plainAt := -1, -1 + for n, line := range strings.Split(string(normalize(data)), "\n") { + if strings.TrimSpace(line) == "" || strings.HasPrefix(line, "#") { + continue + } + indented := strings.HasPrefix(line, " ") + toks, err := tokens(strings.TrimSpace(line)) + if err != nil { + return snapshot{}, fmt.Errorf("line %d: %w", n+1, err) + } + switch { + case !indented && toks[0] == "table" && len(toks) == 2 && !table: + s.table, table = toks[1], true + case !indented && toks[0] == "column" && len(toks) == 2: + s.columns = append(s.columns, column{name: toks[1]}) + colAt, plainAt = len(s.columns)-1, -1 + case !indented && toks[0] == "plaintext" && len(toks) == 2: + s.plaintext = append(s.plaintext, plain{field: toks[1]}) + colAt, plainAt = -1, len(s.plaintext)-1 + case indented && toks[0] == "context" && len(toks) == 2 && colAt >= 0: + s.columns[colAt].context = toks[1] + case indented && toks[0] == "terms" && len(toks) >= 2 && colAt >= 0: + if len(toks) == 2 && toks[1] == "none" { + s.columns[colAt].terms = nil + } else { + s.columns[colAt].terms = toks[1:] + } + case indented && toks[0] == "fact" && len(toks) == 3 && colAt >= 0: + s.columns[colAt].facts = append(s.columns[colAt].facts, fact{toks[1], toks[2]}) + case indented && toks[0] == "fact" && len(toks) == 3 && plainAt >= 0: + s.plaintext[plainAt].facts = append(s.plaintext[plainAt].facts, fact{toks[1], toks[2]}) + default: + return snapshot{}, fmt.Errorf("line %d: unexpected %q", n+1, line) + } + } + if !table { + return snapshot{}, errors.New("no table line") + } + for i := range s.columns { + sortFacts(s.columns[i].facts) + } + for i := range s.plaintext { + sortFacts(s.plaintext[i].facts) + } + s.sort() + return s, nil +} + +// tokens splits a line into its values, unquoting the quoted ones. +func tokens(line string) ([]string, error) { + var out []string + for line != "" { + if line[0] == '"' { + q, err := strconv.QuotedPrefix(line) + if err != nil { + return nil, fmt.Errorf("bad quoted value in %q", line) + } + v, err := strconv.Unquote(q) + if err != nil { + return nil, fmt.Errorf("bad quoted value in %q", line) + } + out = append(out, v) + line = line[len(q):] + } else { + end := strings.IndexByte(line, ' ') + if end < 0 { + end = len(line) + } + out = append(out, line[:end]) + line = line[end:] + } + line = strings.TrimLeft(line, " ") + } + if len(out) == 0 { + return nil, errors.New("empty line") + } + return out, nil +} diff --git a/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/audits.golden b/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/audits.golden new file mode 100644 index 000000000..7c0ccce9f --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/audits.golden @@ -0,0 +1,8 @@ +# Written by plantest.Golden: what the policy stores each field it decides as. +# Regenerate it with go test -update; do not edit it by hand. +# A column's context is bound into every ciphertext and query term written under it, so once a row is written it must never change. + +table audits + +plaintext kind + fact fides.data_categories system.operations diff --git a/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/individuals.golden b/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/individuals.golden new file mode 100644 index 000000000..56e39a3ff --- /dev/null +++ b/languages/golang/stackencrypt/plan/plantest/testdata/TestPolicies/individuals.golden @@ -0,0 +1,28 @@ +# Written by plantest.Golden: what the policy stores each field it decides as. +# Regenerate it with go test -update; do not edit it by hand. +# A column's context is bound into every ciphertext and query term written under it, so once a row is written it must never change. + +table individuals + +column email + context individuals/email + terms eq match + fact fides.data_categories user.contact.email + +column medicare_num + context individuals/medicare_number + terms eq + fact fides.data_categories user.government_id + +column name + context individuals/name + terms ore + fact fides.data_categories user.name + +column notes + context individuals-notes/v1 + terms none + fact fides.data_categories user.content + +plaintext country + fact fides.data_categories system.operations diff --git a/languages/golang/stackencrypt/record.go b/languages/golang/stackencrypt/record.go index e29196535..2f95f1847 100644 --- a/languages/golang/stackencrypt/record.go +++ b/languages/golang/stackencrypt/record.go @@ -79,23 +79,107 @@ type EncryptedField struct { // EncryptedRecord is one record's planned fields, by wire name. type EncryptedRecord map[string]EncryptedField -// RecordOption adjusts how a record call binds its fields. -type RecordOption func(*recordOptions) +// RecordOption adjusts one record call: [Cipher.EncryptRecords], +// [Cipher.EncryptRecord], [Cipher.DecryptRecords], [Cipher.DecryptRecord] +// and the Client forms of the last two. An option is a value built by one +// of the functions below, and a call applies the options it is given in +// order. +// +// A RecordOption that is not also an [Option], such as [WithPlan], means +// something only on a record call, so handing it to [Cipher.Term] does not +// compile. +type RecordOption interface { + applyRecord(*recordOptions) +} + +// Option is a [RecordOption] that the probe call, [Cipher.Term], accepts +// as well: what a record and the probe that matches it must agree on. +// Every Option is a RecordOption, so one value serves the encrypt, decrypt +// and probe calls alike, and the three cannot drift apart: +// +// tenant := stackencrypt.ExtendContext(uint64(tenantID)) +// rows, err := cipher.EncryptRecords(ctx, users, tenant) +// probe, err := cipher.Term(ctx, "bob@example.com", email, stackencrypt.Equality, tenant) +// err = cipher.DecryptRecords(ctx, rows, &back, tenant) +type Option interface { + RecordOption + applyTerm(*termOptions) +} type recordOptions struct { extension []any plan Plan } +type termOptions struct { + extension []any +} + +// contextExtension is what [ExtendContext] returns. +type contextExtension struct{ parts []any } + +// appendTo adds the extension's parts after any an earlier option gave: +// the one rule for combining extensions, shared by the record calls and +// the probe so the two cannot combine them differently. +func (e contextExtension) appendTo(ext *[]any) { + *ext = append(*ext, e.parts...) +} + +func (e contextExtension) applyRecord(o *recordOptions) { e.appendTo(&o.extension) } + +func (e contextExtension) applyTerm(o *termOptions) { e.appendTo(&o.extension) } + // ExtendContext extends every field's context by parts, in order, the way // the Rust derive extends a field's context by the caller's // (encrypt_into_with_context): a field tagged context=users/age with -// ExtendContext(uint64(7)) binds ["users/age", 7]. The same extension must -// be given to decrypt the records. -func ExtendContext(parts ...any) RecordOption { - return func(o *recordOptions) { o.extension = append(o.extension, parts...) } +// ExtendContext(uint64(7)) binds ["users/age", 7]. On [Cipher.Term] it +// extends the probe's context the same way, so a probe built under the +// extension a record was written under compares against that record's +// terms, and under any other extension, or none, against nothing. +// +// The same extension must be given to decrypt the records. A part's type +// is part of the context (an int crosses as int64, so uint64(7) and 7 are +// different contexts), which is why an extension is best held in one value +// and passed to every call rather than spelled afresh at each. The option +// owns its parts: a byte-slice part is copied, so a caller's buffer reused +// after the call does not change what the option extends by. +// +// Several ExtendContext options on one call join in order: +// ExtendContext(a), ExtendContext(b) is the same context as +// ExtendContext(a, b), on a record call and on Term alike. So each call +// must receive a given extension once. A helper that always adds the +// tenant, called by code that adds the tenant as well, writes records +// under [field, tenant, tenant], and a probe built with the tenant once +// matches none of them, with no error. +// +// A part is checked when a call applies it, not here: a part that is not +// a string, a byte slice or an integer fails the call it is given to. +func ExtendContext(parts ...any) Option { + owned := make([]any, len(parts)) + for i, part := range parts { + owned[i] = ownPart(part) + } + return contextExtension{parts: owned} } +// extend is c extended by every part of ext, in order: the one definition +// of how an extension applies, shared by the record plan and the probe so +// the two cannot disagree. +func extend(c Context, ext []any) (Context, error) { + for _, part := range ext { + var err error + if c, err = c.With(part); err != nil { + return Context{}, err + } + } + return c, nil +} + +// planOption is what [WithPlan] returns. +type planOption struct{ plan Plan } + +func (p planOption) applyRecord(o *recordOptions) { o.plan = p.plan } + // WithPlan encrypts or decrypts records under an explicit plan instead of // the struct's `stash` tags. // @@ -108,7 +192,7 @@ func ExtendContext(parts ...any) RecordOption { // with the same Names and Contexts. Terms are one-way outputs, derived on // encryption and never sent to decrypt, so they need not match either. func WithPlan(p Plan) RecordOption { - return func(o *recordOptions) { o.plan = p } + return planOption{plan: p} } // FieldPlan is one planned field of a record. @@ -355,10 +439,8 @@ func planValue(plan []fieldPlan, opts recordOptions) (vcvalue.Object, error) { if err != nil { return nil, err } - for _, part := range opts.extension { - if ctx, err = ctx.With(part); err != nil { - return nil, err - } + if ctx, err = extend(ctx, opts.extension); err != nil { + return nil, err } outputs := make([]any, len(f.outputs)) for i, o := range f.outputs { @@ -375,7 +457,7 @@ func planValue(plan []fieldPlan, opts recordOptions) (vcvalue.Object, error) { func applyOptions(opts []RecordOption) recordOptions { var o recordOptions for _, opt := range opts { - opt(&o) + opt.applyRecord(&o) } return o } diff --git a/languages/golang/stackencrypt/unit_test.go b/languages/golang/stackencrypt/unit_test.go index 12c5536a8..5b3a206fd 100644 --- a/languages/golang/stackencrypt/unit_test.go +++ b/languages/golang/stackencrypt/unit_test.go @@ -128,6 +128,32 @@ func TestSelectorsSpellEveryVariant(t *testing.T) { } } +// A Context owns its parts, as an option does: NewContext and With copy a +// byte-slice part in, so a caller's buffer reused once the context is +// built does not change it. +func TestContextOwnsItsByteParts(t *testing.T) { + root, ext := []byte("users/email"), []byte("eu") + c, err := NewContext(root) + if err != nil { + t.Fatal(err) + } + if c, err = c.With(ext); err != nil { + t.Fatal(err) + } + copy(root, "users/phone") + copy(ext, "us") + parts, ok := c.value().([]any) + if !ok || len(parts) != 2 { + t.Fatalf("context value is %#v, want a two-part list", c.value()) + } + if got := string(parts[0].([]byte)); got != "users/email" { + t.Errorf("root part is %q after the caller's buffer changed, want \"users/email\"", got) + } + if got := string(parts[1].([]byte)); got != "eu" { + t.Errorf("extension part is %q after the caller's buffer changed, want \"eu\"", got) + } +} + func TestContextNestsToTheLeft(t *testing.T) { c := MustContext("users/age") if got := c.value(); got != "users/age" { @@ -276,6 +302,153 @@ func TestExplicitPlanIsTheTagPlan(t *testing.T) { } } +// A probe's context under ExtendContext is, byte for byte, the context the +// record plan sends for a field with the same own context under the same +// extension — the one place the probe and the stored term could silently +// disagree. And it differs from the unextended context and from another +// extension's, which is what makes the match tenant-specific. +func TestTermExtensionMatchesRecordFieldContext(t *testing.T) { + type row struct { + Email string `stash:"context=users/email,index=eq"` + } + ext := []any{uint64(7), "eu"} + o := applyOptions([]RecordOption{ExtendContext(ext...)}) + bound, err := planFor(reflect.TypeOf(row{}), o) + if err != nil { + t.Fatal(err) + } + obj, err := planValue(bound, o) + if err != nil { + t.Fatal(err) + } + spec, ok := obj[0].Value.(vcvalue.Object) + if !ok || spec[0].Key != "context" { + t.Fatalf("plan field encodes as %+v", obj[0].Value) + } + fieldContext := spec[0].Value + + var to termOptions + ExtendContext(ext...).applyTerm(&to) + probe, err := extend(MustContext("users/email"), to.extension) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(probe.value(), fieldContext) { + t.Fatalf("probe context %#v, record field context %#v", probe.value(), fieldContext) + } + if reflect.DeepEqual(MustContext("users/email").value(), fieldContext) { + t.Fatal("the unextended probe context equals the extended field's") + } + other, err := extend(MustContext("users/email"), []any{uint64(8), "eu"}) + if err != nil { + t.Fatal(err) + } + if reflect.DeepEqual(other.value(), fieldContext) { + t.Fatal("another tenant's probe context equals the field's") + } + // The same option value, held once and passed to both calls, is how + // the two sides are kept in step; it applies identically through + // either interface. An option that means something only on a record + // call is not an Option, so Cipher.Term cannot accept it and ignore it. + var opt RecordOption = ExtendContext(ext...) + if _, ok := opt.(Option); !ok { + t.Fatal("ExtendContext is not an Option through its RecordOption interface") + } + if _, ok := WithPlan(Plan{}).(Option); ok { + t.Fatal("WithPlan is an Option; Cipher.Term must not accept it") + } +} + +// Several ExtendContext options on one call join in order, and the record +// calls and the probe join them by the same rule: two options a and b are +// the context ExtendContext(a, b) gives, on both sides. A rule that let a +// later option replace an earlier one on one side only would put records +// and probes under different contexts with no error. +func TestSeveralExtensionsJoinInOrder(t *testing.T) { + type row struct { + Email string `stash:"context=users/email,index=eq"` + } + typ := reflect.TypeOf(row{}) + fieldContext := func(opts ...RecordOption) any { + t.Helper() + o := applyOptions(opts) + bound, err := planFor(typ, o) + if err != nil { + t.Fatal(err) + } + obj, err := planValue(bound, o) + if err != nil { + t.Fatal(err) + } + return obj[0].Value.(vcvalue.Object)[0].Value + } + probeContext := func(opts ...Option) any { + t.Helper() + var to termOptions + for _, opt := range opts { + opt.applyTerm(&to) + } + c, err := extend(MustContext("users/email"), to.extension) + if err != nil { + t.Fatal(err) + } + return c.value() + } + + tenant, region := ExtendContext(uint64(7)), ExtendContext("eu") + want := fieldContext(ExtendContext(uint64(7), "eu")) + if got := fieldContext(tenant, region); !reflect.DeepEqual(got, want) { + t.Errorf("record: two options give %#v, one option with both parts %#v", got, want) + } + if got := probeContext(tenant, region); !reflect.DeepEqual(got, want) { + t.Errorf("probe: two options give %#v, the record's one-option context %#v", got, want) + } + if got := probeContext(ExtendContext(uint64(7), "eu")); !reflect.DeepEqual(got, want) { + t.Errorf("probe: one option gives %#v, the record's %#v", got, want) + } + // Order is part of the context: the same parts the other way round are + // another context, on both sides. + if got := fieldContext(region, tenant); reflect.DeepEqual(got, want) { + t.Error("record: options in the other order give the same context") + } + if got := probeContext(region, tenant); reflect.DeepEqual(got, want) { + t.Error("probe: options in the other order give the same context") + } + // Joining is not deduplication: the same extension given twice extends + // twice, which is why a call must receive it once. + if got := fieldContext(tenant, tenant); reflect.DeepEqual(got, fieldContext(tenant)) { + t.Error("record: the same extension given twice is the single-extension context") + } +} + +// An option owns its parts. A byte-slice part is copied when the option +// is built, so a caller's buffer reused between the write and the probe +// does not move the context the saved option extends by, on either side. +func TestExtendContextOwnsItsByteParts(t *testing.T) { + region := []byte("eu") + opt := ExtendContext(uint64(7), region) + first := applyOptions([]RecordOption{opt}) + var firstProbe termOptions + opt.applyTerm(&firstProbe) + + copy(region, "us") + + second := applyOptions([]RecordOption{opt}) + var secondProbe termOptions + opt.applyTerm(&secondProbe) + for name, ext := range map[string][]any{ + "record, before": first.extension, "record, after": second.extension, + "probe, before": firstProbe.extension, "probe, after": secondProbe.extension, + } { + if got := string(ext[1].([]byte)); got != "eu" { + t.Errorf("%s: byte part is %q after the caller's buffer changed, want \"eu\"", name, got) + } + } + if !reflect.DeepEqual(first.extension, second.extension) || !reflect.DeepEqual(firstProbe.extension, secondProbe.extension) { + t.Error("the same option applied twice gave different extensions") + } +} + // A plan can name only exported, direct fields of the struct it binds to, // and only fields that exist; an untagged struct binds fine under it. func TestPlanBindsByFieldName(t *testing.T) {