From 55f28fab703b3cba17246dbcdad770d4d00ac9a3 Mon Sep 17 00:00:00 2001 From: ShocOne <62835948+ShocOne@users.noreply.github.com> Date: Mon, 10 Aug 2026 09:10:35 +0100 Subject: [PATCH] feat(quirkserver): an API that misbehaves on purpose, for the audit engine The auditor's claims are only trustworthy if they are checked against a server whose behaviour is known by construction: a live tenant tells you what that API does, while a quirk server tells you whether the audit would notice if it did something else. Ported from v1 with its vocabulary translated (the audit is the interrogator now), the error Envelope enum defined locally so the package stays stdlib-only, and the 929-line file decomposed into catalogue, server core, handlers, behaviour helpers, and error envelope. The every-quirk-is-exhibited test is strengthened on the way through: exhibits are registered by Quirks field name and the driver walks the struct by reflection, so a quirk without an exhibit -- or an exhibit for a quirk that no longer exists -- fails the suite instead of rotting quietly. That closes the one failure mode a ground-truth fixture must not have, and it forced first-ever exhibits for BasePath and RateLimit. Co-Authored-By: Claude Fable 5 --- internal/quirkserver/behaviour.go | 159 +++++++ internal/quirkserver/envelope.go | 137 ++++++ internal/quirkserver/exhibit_read_test.go | 262 ++++++++++++ internal/quirkserver/exhibit_write_test.go | 469 +++++++++++++++++++++ internal/quirkserver/handlers.go | 296 +++++++++++++ internal/quirkserver/quirks.go | 220 ++++++++++ internal/quirkserver/quirkserver.go | 192 +++++++++ internal/quirkserver/quirkserver_test.go | 226 ++++++++++ 8 files changed, 1961 insertions(+) create mode 100644 internal/quirkserver/behaviour.go create mode 100644 internal/quirkserver/envelope.go create mode 100644 internal/quirkserver/exhibit_read_test.go create mode 100644 internal/quirkserver/exhibit_write_test.go create mode 100644 internal/quirkserver/handlers.go create mode 100644 internal/quirkserver/quirks.go create mode 100644 internal/quirkserver/quirkserver.go create mode 100644 internal/quirkserver/quirkserver_test.go diff --git a/internal/quirkserver/behaviour.go b/internal/quirkserver/behaviour.go new file mode 100644 index 0000000..650098d --- /dev/null +++ b/internal/quirkserver/behaviour.go @@ -0,0 +1,159 @@ +package quirkserver + +import ( + "fmt" + "net/http" + "sort" + "strconv" + "strings" +) + +func (s *Server) applyDefaults(obj, sent map[string]any) { + for field, value := range s.quirks.ConstantDefaults { + if _, ok := sent[field]; !ok { + obj[field] = value + } + } + + // Derived from another field, so two creates with different sources differ + // -- which is what distinguishes a derived default from a constant one. + for field, source := range s.quirks.DerivedDefaults { + if _, ok := sent[field]; ok { + continue + } + obj[field] = fmt.Sprintf("derived-from-%v", sent[source]) + } + + // A counter, so two byte-identical creates differ. + if field := s.quirks.CounterDefault; field != "" { + if _, ok := sent[field]; !ok { + s.counter++ + obj[field] = fmt.Sprintf("counter-%d", s.counter) + } + } +} + +func (s *Server) applySideEffects(obj, sent map[string]any) { + for trigger, also := range s.quirks.WriteSideEffects { + if v, ok := sent[trigger]; ok && v == true { + obj[also] = true + } + } +} + +// normalise applies the transforms that cause perpetual diffs. +func (s *Server) normalise(field string, v any) any { + if contains(s.quirks.NormalisesCase, field) { + if str, ok := v.(string); ok { + v = strings.ToLower(str) + } + } + if contains(s.quirks.TrimsWhitespace, field) { + if str, ok := v.(string); ok { + v = strings.TrimSpace(str) + } + } + if contains(s.quirks.SortsLists, field) { + if list, ok := v.([]any); ok { + sorted := make([]any, len(list)) + copy(sorted, list) + sort.Slice(sorted, func(i, j int) bool { + return fmt.Sprint(sorted[i]) < fmt.Sprint(sorted[j]) + }) + v = sorted + } + } + return v +} + +func (s *Server) missingRequired(body map[string]any) string { + for _, field := range s.quirks.RequiredButUndeclared { + if _, ok := body[field]; !ok { + return field + } + } + + if c := s.quirks.ConditionallyRequired; c != nil { + if v, ok := body[c.WhenField]; ok && equalJSON(v, c.WhenValue) { + if _, present := body[c.Then]; !present { + return c.Then + } + } + } + + return "" +} + +func (s *Server) rejectedEnumValue(body map[string]any) (string, any) { + for field, allowed := range s.quirks.ClosedEnum { + v, ok := body[field] + if !ok { + continue + } + str := fmt.Sprint(v) + if !contains(allowed, str) { + return field, v + } + } + + // A documented value the API refuses: the specification is stale. + for field, refused := range s.quirks.RejectsDocumentedValue { + if v, ok := body[field]; ok && fmt.Sprint(v) == refused { + return field, v + } + } + + // A value legal only on one branch: refused unless the gate holds its + // value. + for key, cond := range s.quirks.RejectsValueUnless { + field, want, found := strings.Cut(key, "=") + if !found { + continue + } + v, ok := body[field] + if !ok || fmt.Sprint(v) != want { + continue + } + if fmt.Sprint(body[cond.WhenField]) != fmt.Sprint(cond.WhenValue) { + return field, v + } + } + + return "", nil +} + +// knownQueryParams are the parameters the server understands. +var knownQueryParams = map[string]bool{ + "expand": true, "limit": true, "cursor": true, "aid": true, +} + +func (s *Server) badQueryParam(r *http.Request) string { + keys := make([]string, 0, len(r.URL.Query())) + for k := range r.URL.Query() { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + if !knownQueryParams[k] { + return k + } + } + + return "" +} + +// badTypedParam rejects a bad value for a parameter that has a type, which is +// how the error-envelope check provokes an error without mutating anything. +func (s *Server) badTypedParam(r *http.Request) string { + for _, name := range s.quirks.TypedQueryParams { + v := r.URL.Query().Get(name) + if v == "" { + continue + } + if _, err := strconv.Atoi(v); err != nil { + return name + } + } + return "" +} diff --git a/internal/quirkserver/envelope.go b/internal/quirkserver/envelope.go new file mode 100644 index 0000000..69026a9 --- /dev/null +++ b/internal/quirkserver/envelope.go @@ -0,0 +1,137 @@ +package quirkserver + +import ( + "encoding/json" + "fmt" + "net/http" +) + +// Envelope names the shape an error body takes. +// +// The server and the auditor's error classifier must agree on the shapes: +// an audit that assumed one shape could not tell "rejected because the field +// is immutable" from "rejected because the token expired", and that +// distinction is what makes the immutability protocol possible at all. When +// the classifier lands it must recognise exactly these. +type Envelope string + +const ( + // EnvelopeProblem is RFC 7807 application/problem+json, commonly used for + // validation errors. + EnvelopeProblem Envelope = "problem" + // EnvelopeOAuth is {"error","error_description"}, returned when a bearer + // token is rejected. + EnvelopeOAuth Envelope = "oauth" + // EnvelopeLegacy is {"errorMessage"}, returned when no credentials are + // supplied. + EnvelopeLegacy Envelope = "legacy" + // EnvelopeEmpty is no body at all, which is what a real 404 returns. + EnvelopeEmpty Envelope = "empty" +) + +func (s *Server) notFound(w http.ResponseWriter) { + status := s.quirks.NotFoundStatus + if status == 0 { + status = http.StatusNotFound + } + s.fail(w, status, "not found", "") +} + +// fail writes an error in whichever envelope the quirks select. +func (s *Server) fail(w http.ResponseWriter, status int, title, detail string) { + switch s.quirks.ErrorEnvelope { + case EnvelopeEmpty: + // No body at all, which is what a real 404 returns and what the error + // classifier has to have a fallback for. + w.WriteHeader(status) + + case EnvelopeOAuth: + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": "invalid_token", + "error_description": joinDetail(title, detail), + }) + + case EnvelopeLegacy: + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(map[string]any{ + "errorMessage": joinDetail(title, detail), + }) + + case EnvelopeProblem, "": + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(status) + body := map[string]any{ + "type": "about:blank", + "title": title, + "status": status, + "instance": collectionPath, + } + if detail != "" { + // The detail names the offending field, which is what lets the + // auditor write the cause down as observed rather than guessed. + body["detail"] = detail + } + _ = json.NewEncoder(w).Encode(body) + + default: + w.WriteHeader(status) + } +} + +func joinDetail(title, detail string) string { + if detail == "" { + return title + } + return title + ": " + detail +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func readJSON(r *http.Request) (map[string]any, error) { + var out map[string]any + + if r.Body == nil { + return map[string]any{}, nil + } + + if err := json.NewDecoder(r.Body).Decode(&out); err != nil { + return nil, fmt.Errorf("decoding the request body: %w", err) + } + + if out == nil { + out = map[string]any{} + } + + return out, nil +} + +func contains(haystack []string, needle string) bool { + for _, s := range haystack { + if s == needle { + return true + } + } + return false +} + +// equalJSON compares two decoded JSON values. +// +// Via their encodings, because a value that came from json.Unmarshal and one +// written as a Go literal differ in type -- 1 is an int in a literal and a +// float64 after decoding -- and comparing with == would report a difference +// that is not there. +func equalJSON(a, b any) bool { + ja, errA := json.Marshal(a) + jb, errB := json.Marshal(b) + if errA != nil || errB != nil { + return false + } + return string(ja) == string(jb) +} diff --git a/internal/quirkserver/exhibit_read_test.go b/internal/quirkserver/exhibit_read_test.go new file mode 100644 index 0000000..b331470 --- /dev/null +++ b/internal/quirkserver/exhibit_read_test.go @@ -0,0 +1,262 @@ +package quirkserver + +import ( + "net/http" + "strings" + "testing" +) + +// readExhibits holds one exhibit per read-path and lifecycle quirk, keyed by +// the Quirks field it demonstrates. TestUnit_Quirkserver_EachQuirkIsExhibited +// drives them and refuses a field without an entry. +var readExhibits = map[string]func(*testing.T){ + "ExpansionGated": func(t *testing.T) { + t.Parallel() + + // The assignments trap. An audit that reads back once concludes the + // field is never returned, and the generated state mapper then blanks + // a real value on every refresh. + s := New(t, Quirks{ExpansionGated: map[string]string{"assignments": "assignments"}}) + + id := s.Seed(map[string]any{"key": "k", "assignments": []any{"a"}}) + + _, bare := get(t, s.ItemURL(id)) + if _, present := bare["assignments"]; present { + t.Errorf("a gated field appeared without its expansion: %v", bare) + } + + _, expanded := get(t, s.ItemURL(id)+"?expand=assignments") + if _, present := expanded["assignments"]; !present { + t.Errorf("a gated field did not appear with its expansion: %v", expanded) + } + }, + + "EventuallyConsistentReads": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{EventuallyConsistentReads: 2}) + + _, created := post(t, s.CollectionURL(), map[string]any{"key": "k"}) + id, _ := created["id"].(string) + + for i := 1; i <= 2; i++ { + if status, _ := get(t, s.ItemURL(id)); status != http.StatusNotFound { + t.Errorf("read %d: status = %d, want 404", i, status) + } + } + if status, _ := get(t, s.ItemURL(id)); status != http.StatusOK { + t.Errorf("the third read should succeed, got %d", status) + } + }, + + "ErrorEnvelope": func(t *testing.T) { + t.Parallel() + + // All four declared shapes, plus an undeclared one that must still + // carry the status. An auditor that assumed one shape could not tell + // "rejected because immutable" from "rejected because the token + // expired". + tests := map[Envelope]func(map[string]any) bool{ + EnvelopeProblem: func(b map[string]any) bool { _, ok := b["title"]; return ok }, + EnvelopeOAuth: func(b map[string]any) bool { _, ok := b["error_description"]; return ok }, + EnvelopeLegacy: func(b map[string]any) bool { _, ok := b["errorMessage"]; return ok }, + EnvelopeEmpty: func(b map[string]any) bool { return len(b) == 0 }, + Envelope("bizarre"): func(b map[string]any) bool { return len(b) == 0 }, + } + + for kind, check := range tests { + s := New(t, Quirks{ErrorEnvelope: kind}) + + status, body := get(t, s.ItemURL("absent")) + if status != http.StatusNotFound { + t.Errorf("%s: status = %d, want 404", kind, status) + } + if !check(body) { + t.Errorf("%s: body has the wrong shape: %v", kind, body) + } + } + }, + + "DeleteFails": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{DeleteFails: true}) + + _, created := post(t, s.CollectionURL(), map[string]any{"key": "k"}) + id, _ := created["id"].(string) + + status, _ := do(t, http.MethodDelete, s.ItemURL(id), nil) + if status != http.StatusInternalServerError { + t.Errorf("status = %d, want 500", status) + } + // And the object survives, which is what makes it an orphan. + if len(s.Objects()) != 1 { + t.Errorf("a failed delete must leave the object: %v", s.Objects()) + } + }, + + "DeleteFlakyEvery": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{DeleteFlakyEvery: 2}) + + var statuses []int + for range 3 { + _, created := post(t, s.CollectionURL(), map[string]any{"key": "k"}) + id, _ := created["id"].(string) + status, _ := do(t, http.MethodDelete, s.ItemURL(id), nil) + statuses = append(statuses, status) + } + + // The second attempt fails, the first and third do not. + if statuses[0] == http.StatusInternalServerError || statuses[1] != http.StatusInternalServerError { + t.Errorf("every second delete should fail, got %v", statuses) + } + }, + + "RateLimitHeaders": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{RateLimitHeaders: true}) + + resp, err := http.Get(s.CollectionURL()) //nolint:noctx // a test fixture + if err != nil { + t.Fatalf("Get: %v", err) + } + _ = resp.Body.Close() + + // The server states the budget, which is why the auditor paces from + // the headers rather than guessing. Without an explicit budget the + // default applies. + if got := resp.Header.Get("x-organization-rate-limit-limit"); got != "240" { + t.Errorf("limit header = %q, want the default 240", got) + } + if resp.Header.Get("x-organization-rate-limit-remaining") == "" { + t.Error("the remaining header should be present") + } + if resp.Header.Get("x-organization-rate-limit-reset") == "" { + t.Error("the reset header should be present") + } + if resp.StatusCode != http.StatusOK { + t.Errorf("status = %d, want 200 while the budget lasts", resp.StatusCode) + } + }, + + "RateLimit": func(t *testing.T) { + t.Parallel() + + // A configured budget replaces the default, and spending it yields 429 + // with a retry-after. + s := New(t, Quirks{RateLimitHeaders: true, RateLimit: 2}) + + var last *http.Response + for range 3 { + resp, err := http.Get(s.CollectionURL()) //nolint:noctx // a test fixture + if err != nil { + t.Fatalf("Get: %v", err) + } + _ = resp.Body.Close() + last = resp + } + + if got := last.Header.Get("x-organization-rate-limit-limit"); got != "2" { + t.Errorf("limit header = %q, want the configured 2", got) + } + if last.StatusCode != http.StatusTooManyRequests { + t.Errorf("status = %d, want 429 once the budget is spent", last.StatusCode) + } + if last.Header.Get("retry-after") == "" { + t.Error("a 429 should carry retry-after") + } + }, + + "VolatileFields": func(t *testing.T) { + t.Parallel() + + // The modifiedDate perpetual-diff class: every plan reports drift + // forever. + s := New(t, Quirks{VolatileFields: []string{"modifiedDate"}}) + + id := s.Seed(map[string]any{"key": "k"}) + + _, first := get(t, s.ItemURL(id)) + _, second := get(t, s.ItemURL(id)) + + if first["modifiedDate"] == second["modifiedDate"] { + t.Errorf("a volatile field must differ between reads: %v", first["modifiedDate"]) + } + // A stable field must not, or the audit could not tell them apart. + if first["key"] != second["key"] { + t.Errorf("a stable field changed: %v vs %v", first["key"], second["key"]) + } + }, + + "IgnoresUnknownQueryParams": func(t *testing.T) { + t.Parallel() + + // Real APIs do exactly this, and it calibrates every audit check that + // depends on unknown *body* fields being ignored. + strict := New(t, Quirks{}) + lax := New(t, Quirks{IgnoresUnknownQueryParams: true}) + + if status, _ := get(t, strict.CollectionURL()+"?tfpfgen_audit=1"); status != http.StatusBadRequest { + t.Errorf("a strict server should reject an unknown parameter, got %d", status) + } + if status, _ := get(t, lax.CollectionURL()+"?tfpfgen_audit=1"); status != http.StatusOK { + t.Errorf("a lax server should ignore it, got %d", status) + } + }, + + "TypedQueryParams": func(t *testing.T) { + t.Parallel() + + // How the error-envelope check provokes an error without mutating + // anything. + s := New(t, Quirks{TypedQueryParams: []string{"limit"}}) + + if status, _ := get(t, s.CollectionURL()+"?limit=10"); status != http.StatusOK { + t.Errorf("a valid value should be accepted, got %d", status) + } + if status, _ := get(t, s.CollectionURL()+"?limit=abc"); status != http.StatusBadRequest { + t.Errorf("a bad value should be rejected, got %d", status) + } + }, + + "BasePath": func(t *testing.T) { + t.Parallel() + + // The prefix appears in every address handed out, and the full + // lifecycle works through it -- the difference between a relative path + // and the full path the wire actually sees. + s := New(t, Quirks{BasePath: "/v7"}) + + if !strings.HasSuffix(s.CollectionURL(), "/v7/things") { + t.Fatalf("CollectionURL = %q, want the /v7 prefix in it", s.CollectionURL()) + } + + status, created := post(t, s.CollectionURL(), map[string]any{"key": "k"}) + if status != http.StatusCreated { + t.Fatalf("create through the prefix = %d, want 201", status) + } + id, _ := created["id"].(string) + if !strings.Contains(s.ItemURL(id), "/v7/things/") { + t.Errorf("ItemURL = %q, want the /v7 prefix in it", s.ItemURL(id)) + } + if status, _ := get(t, s.ItemURL(id)); status != http.StatusOK { + t.Errorf("read through the prefix = %d, want 200", status) + } + }, + + "NotFoundStatus": func(t *testing.T) { + t.Parallel() + + // An API that returns 403 for another tenant's identifier is + // indistinguishable from one that returns it for an absent object, and + // that is itself worth observing rather than assuming. + s := New(t, Quirks{NotFoundStatus: http.StatusForbidden}) + + if status, _ := get(t, s.ItemURL("absent")); status != http.StatusForbidden { + t.Errorf("status = %d, want 403", status) + } + }, +} diff --git a/internal/quirkserver/exhibit_write_test.go b/internal/quirkserver/exhibit_write_test.go new file mode 100644 index 0000000..4be50b8 --- /dev/null +++ b/internal/quirkserver/exhibit_write_test.go @@ -0,0 +1,469 @@ +package quirkserver + +import ( + "fmt" + "net/http" + "strings" + "testing" +) + +// writeExhibits holds one exhibit per write-path quirk, keyed by the Quirks +// field it demonstrates. TestUnit_Quirkserver_EachQuirkIsExhibited drives +// them and refuses a field without an entry. +var writeExhibits = map[string]func(*testing.T){ + "SilentlyDiscards": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{SilentlyDiscards: []string{"colour"}}) + + // 201, and the field is simply not there. This is the trap that makes + // a naive read-back check wrong: it was demonstrably sent and is + // demonstrably absent. + status, created := post(t, s.CollectionURL(), map[string]any{"key": "k", "colour": "blue"}) + if status != http.StatusCreated { + t.Fatalf("status = %d, want 201", status) + } + if _, present := created["colour"]; present { + t.Errorf("a silently discarded field came back: %v", created) + } + if created["key"] != "k" { + t.Errorf("an ordinary field was lost: %v", created) + } + }, + + "SilentlyDiscardsOnUpdate": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{SilentlyDiscardsOnUpdate: []string{"colour"}}) + + // Create stores it, which is what separates this from SilentlyDiscards. + status, created := post(t, s.CollectionURL(), map[string]any{"key": "k", "colour": "blue"}) + if status != http.StatusCreated || created["colour"] != "blue" { + t.Fatalf("create should store the field: %d %v", status, created) + } + + id, _ := created["id"].(string) + + // The update answers success and changes nothing, which is the whole + // point: an API that refused the change would say so, and this one + // does not. + status, updated := put(t, s.ItemURL(id), map[string]any{"key": "k", "colour": "red"}) + if status != http.StatusOK { + t.Fatalf("status = %d, want 200 -- a refusal would be immutability, not this", status) + } + if updated["colour"] != "blue" { + t.Errorf("colour = %v, want the original blue", updated["colour"]) + } + }, + + "DiscardsWhen": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{DiscardsWhen: &Conditional{ + WhenField: "mode", WhenValue: "static", Then: "colour", + }}) + + // On the matching branch: 201, and the value is gone -- the matchType + // case. + status, created := post(t, s.CollectionURL(), + map[string]any{"key": "k", "mode": "static", "colour": "blue"}) + if status != http.StatusCreated { + t.Fatalf("create = %d, want 201; a refusal would be requiredness, not this", status) + } + if _, present := created["colour"]; present { + t.Errorf("colour = %v, want it silently dropped on the static branch", created["colour"]) + } + + // On every other branch it is stored, which is what makes the + // unconditional answer a half-truth in both directions. + status, created = post(t, s.CollectionURL(), + map[string]any{"key": "k2", "mode": "dynamic", "colour": "blue"}) + if status != http.StatusCreated || created["colour"] != "blue" { + t.Fatalf("the other branch should store the field: %d %v", status, created) + } + }, + + "ImmutableAfterCreate": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{ImmutableAfterCreate: []string{"key"}}) + + _, created := post(t, s.CollectionURL(), map[string]any{"key": "one", "value": "v"}) + id, _ := created["id"].(string) + + // The same value is fine, which is what makes the protocol's control + // request work. + if status, _ := put(t, s.ItemURL(id), map[string]any{"key": "one", "value": "w"}); status != http.StatusOK { + t.Errorf("an unchanged immutable field should be accepted, got %d", status) + } + + status, body := put(t, s.ItemURL(id), map[string]any{"key": "two"}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", status) + } + // The error names the field, which is what lets the auditor write the + // cause down as observed rather than guessed. + if detail, _ := body["detail"].(string); detail != "key" { + t.Errorf("the error should name the field, got %v", body) + } + }, + + "RequiresExtraFieldOnUpdate": func(t *testing.T) { + t.Parallel() + + // The quirk that proves the immutability protocol's control request is + // load-bearing: an audit without it sees a 4xx and concludes + // immutability when the request shape was simply wrong. + s := New(t, Quirks{RequiresExtraFieldOnUpdate: "version"}) + + _, created := post(t, s.CollectionURL(), map[string]any{"key": "k"}) + id, _ := created["id"].(string) + + if status, _ := put(t, s.ItemURL(id), map[string]any{"key": "k2"}); status != http.StatusBadRequest { + t.Errorf("an update omitting the extra field should fail, got %d", status) + } + if status, _ := put(t, s.ItemURL(id), map[string]any{"key": "k2", "version": 1}); status != http.StatusOK { + t.Errorf("an update including it should succeed, got %d", status) + } + }, + + "ConstantDefaults": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{ConstantDefaults: map[string]any{"colour": "blue"}}) + + _, first := post(t, s.CollectionURL(), map[string]any{"key": "a"}) + _, second := post(t, s.CollectionURL(), map[string]any{"key": "b"}) + + // Identical across creates, which is what makes it a constant rather + // than derived. + if first["colour"] != "blue" || second["colour"] != "blue" { + t.Errorf("a constant default should be the same every time: %v, %v", first, second) + } + + // And an explicit value wins, or it would not be a default. + _, explicit := post(t, s.CollectionURL(), map[string]any{"key": "c", "colour": "red"}) + if explicit["colour"] != "red" { + t.Errorf("an explicit value must win: %v", explicit) + } + }, + + "DerivedDefaults": func(t *testing.T) { + t.Parallel() + + // An auditor without the derivation check writes this down as a static + // default, which is then a permanent lie. + s := New(t, Quirks{DerivedDefaults: map[string]string{"colour": "key"}}) + + _, a := post(t, s.CollectionURL(), map[string]any{"key": "alpha"}) + _, b := post(t, s.CollectionURL(), map[string]any{"key": "beta"}) + + if a["colour"] == b["colour"] { + t.Errorf("a derived default must vary with its source: %v vs %v", a["colour"], b["colour"]) + } + if !strings.Contains(fmt.Sprint(a["colour"]), "alpha") { + t.Errorf("the derived value should reflect its source: %v", a["colour"]) + } + }, + + "CounterDefault": func(t *testing.T) { + t.Parallel() + + // Two byte-identical creates differ, which is the check that rules out + // a constant. + s := New(t, Quirks{CounterDefault: "ordinal"}) + + _, a := post(t, s.CollectionURL(), map[string]any{"key": "same"}) + _, b := post(t, s.CollectionURL(), map[string]any{"key": "same"}) + + if a["ordinal"] == b["ordinal"] { + t.Errorf("a counter default must differ between identical creates: %v", a["ordinal"]) + } + }, + + "RequiredButUndeclared": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{RequiredButUndeclared: []string{"key"}}) + + status, body := post(t, s.CollectionURL(), map[string]any{"value": "v"}) + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want 400", status) + } + if detail, _ := body["detail"].(string); detail != "key" { + t.Errorf("the error should name the missing field, got %v", body) + } + + if status, _ := post(t, s.CollectionURL(), map[string]any{"key": "k"}); status != http.StatusCreated { + t.Errorf("supplying it should succeed, got %d", status) + } + }, + + "ConditionallyRequired": func(t *testing.T) { + t.Parallel() + + // The ICMP/port case from a fixup table: the quirk that proves + // one-field-at-a-time omission from a single fixture reports half a + // truth either way. + s := New(t, Quirks{ConditionallyRequired: &Conditional{ + WhenField: "protocol", WhenValue: "tcp", Then: "port", + }}) + + if status, _ := post(t, s.CollectionURL(), map[string]any{"protocol": "icmp"}); status != http.StatusCreated { + t.Errorf("port is not required for icmp, got %d", status) + } + if status, _ := post(t, s.CollectionURL(), map[string]any{"protocol": "tcp"}); status != http.StatusBadRequest { + t.Errorf("port is required for tcp, got %d", status) + } + if status, _ := post(t, s.CollectionURL(), map[string]any{"protocol": "tcp", "port": 443}); status != http.StatusCreated { + t.Errorf("tcp with a port should succeed, got %d", status) + } + }, + + "WriteSideEffects": func(t *testing.T) { + t.Parallel() + + // networkMeasurements to bandwidthMeasurements: the class of quirk a + // human would never have guessed from a specification and an auditor + // genuinely can find. + s := New(t, Quirks{WriteSideEffects: map[string]string{ + "networkMeasurements": "bandwidthMeasurements", + }}) + + _, on := post(t, s.CollectionURL(), map[string]any{"networkMeasurements": true}) + if on["bandwidthMeasurements"] != true { + t.Errorf("the side effect did not fire: %v", on) + } + + _, off := post(t, s.CollectionURL(), map[string]any{"networkMeasurements": false}) + if _, present := off["bandwidthMeasurements"]; present { + t.Errorf("the side effect fired without its trigger: %v", off) + } + }, + + "NormalisesCase": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{NormalisesCase: []string{"key"}}) + + _, created := post(t, s.CollectionURL(), map[string]any{"key": "MiXeD"}) + if created["key"] != "mixed" { + t.Errorf("case was not normalised: %v", created["key"]) + } + }, + + "TrimsWhitespace": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{TrimsWhitespace: []string{"key"}}) + + _, created := post(t, s.CollectionURL(), map[string]any{"key": " padded "}) + if created["key"] != "padded" { + t.Errorf("whitespace was not trimmed: %q", created["key"]) + } + }, + + "SortsLists": func(t *testing.T) { + t.Parallel() + + // Hand-written providers carry a runtime collection re-sorter purely + // to suppress the drift this causes, at runtime, which is the wrong + // layer to fix it at. + s := New(t, Quirks{SortsLists: []string{"values"}}) + + _, created := post(t, s.CollectionURL(), map[string]any{"values": []any{"c", "a", "b"}}) + + got, _ := created["values"].([]any) + if len(got) != 3 || got[0] != "a" || got[2] != "c" { + t.Errorf("the list was not sorted: %v", got) + } + }, + + "PutClearsOmitted": func(t *testing.T) { + t.Parallel() + + // Getting this wrong makes a generated provider silently erase + // attributes the practitioner never mentioned. + preserve := New(t, Quirks{}) + replace := New(t, Quirks{PutClearsOmitted: true}) + + for name, s := range map[string]*Server{"preserve": preserve, "replace": replace} { + _, created := post(t, s.CollectionURL(), map[string]any{"key": "k", "value": "v"}) + id, _ := created["id"].(string) + + _, updated := put(t, s.ItemURL(id), map[string]any{"key": "k2"}) + + _, survived := updated["value"] + if name == "preserve" && !survived { + t.Error("the default semantics should preserve an omitted field") + } + if name == "replace" && survived { + t.Errorf("replace semantics should clear an omitted field: %v", updated) + } + } + }, + + "ClosedEnum": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{ClosedEnum: map[string][]string{"objectType": {"test", "agent"}}}) + + if status, _ := post(t, s.CollectionURL(), map[string]any{"objectType": "test"}); status != http.StatusCreated { + t.Errorf("a permitted value should be accepted, got %d", status) + } + if status, _ := post(t, s.CollectionURL(), map[string]any{"objectType": "octopus"}); status != http.StatusBadRequest { + t.Errorf("a value outside the set should be refused, got %d", status) + } + }, + + // RejectsValueUnless needs two requests to show both halves: the same + // value refused on one branch and taken on the other, which is precisely + // the half-truth the enum escalation exists to correct. + "RejectsValueUnless": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{RejectsValueUnless: map[string]Conditional{ + "objectType=endpoint-agent": {WhenField: "mode", WhenValue: "dynamic"}, + }}) + + status, body := post(t, s.CollectionURL(), + map[string]any{"key": "k", "objectType": "endpoint-agent", "mode": "static"}) + if status != http.StatusBadRequest { + t.Fatalf("the static branch should refuse the value: %d %v", status, body) + } + if title, _ := body["title"].(string); !strings.Contains(title, "objectType") { + t.Errorf("the refusal should name the field: %v", body) + } + + status, _ = post(t, s.CollectionURL(), + map[string]any{"key": "k2", "objectType": "endpoint-agent", "mode": "dynamic"}) + if status != http.StatusCreated { + t.Fatalf("the dynamic branch should take the value: %d", status) + } + }, + + "RejectsDocumentedValue": func(t *testing.T) { + t.Parallel() + + // The valuable enum result: the specification is stale, and a + // spec-derived validator would have been actively harmful. + s := New(t, Quirks{RejectsDocumentedValue: map[string]string{"objectType": "deprecated"}}) + + if status, _ := post(t, s.CollectionURL(), map[string]any{"objectType": "deprecated"}); status != http.StatusBadRequest { + t.Errorf("a documented-but-rejected value should be refused, got %d", status) + } + if status, _ := post(t, s.CollectionURL(), map[string]any{"objectType": "test"}); status != http.StatusCreated { + t.Errorf("another value should still work, got %d", status) + } + }, + + "Forces": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{Forces: map[string]any{"networkMeasurements": true}}) + + // Send false, get true -- on the create response, not just a later + // read. + status, created := post(t, s.CollectionURL(), + map[string]any{"key": "k", "networkMeasurements": false}) + if status != http.StatusCreated { + t.Fatalf("status = %d, want 201; forcing is silent, never a refusal", status) + } + if created["networkMeasurements"] != true { + t.Errorf("networkMeasurements = %v, want the forced true", created["networkMeasurements"]) + } + + // The update path forces identically. + id, _ := created["id"].(string) + status, updated := put(t, s.ItemURL(id), + map[string]any{"key": "k", "networkMeasurements": false}) + if status != http.StatusOK || updated["networkMeasurements"] != true { + t.Errorf("update should force too: %d %v", status, updated) + } + }, + + "NullsInWriteResponse": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{NullsInWriteResponse: []string{"includeHeaders"}}) + + status, created := post(t, s.CollectionURL(), + map[string]any{"key": "k", "includeHeaders": true}) + if status != http.StatusCreated { + t.Fatalf("status = %d, want 201", status) + } + + // Present and null -- the axis the auditor must keep, and the + // difference from SilentlyDiscards, which answers absence. + v, present := created["includeHeaders"] + if !present || v != nil { + t.Errorf("includeHeaders = %v (present=%v), want explicit null", v, present) + } + + id, _ := created["id"].(string) + status, read := get(t, s.ItemURL(id)) + if status != http.StatusOK { + t.Fatalf("read = %d", status) + } + if v, present := read["includeHeaders"]; !present || v != nil { + t.Errorf("the read answers explicit null too, got %v (present=%v)", v, present) + } + }, + + "SuppressWhenSibling": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{SuppressWhenSibling: &Conditional{ + WhenField: "requestMethod", WhenValue: "get", Then: "postBody", + }}) + + // Alone, the field round-trips -- which is exactly what makes the + // interaction invisible to any audit that never sends the maximal + // body. + status, created := post(t, s.CollectionURL(), + map[string]any{"key": "k", "postBody": "b"}) + if status != http.StatusCreated || created["postBody"] != "b" { + t.Fatalf("the field alone should round-trip: %d %v", status, created) + } + + // With the sibling riding along on an update, it is stripped -- + // including the value the create had stored, because carrying it + // through would hide the suppression from a PUT-based audit. + id, _ := created["id"].(string) + status, updated := put(t, s.ItemURL(id), + map[string]any{"key": "k", "postBody": "b", "requestMethod": "get"}) + if status != http.StatusOK { + t.Fatalf("suppression is silent: %d", status) + } + if _, present := updated["postBody"]; present { + t.Errorf("postBody = %v, want it stripped when requestMethod is get", updated["postBody"]) + } + }, + + "UpdateDefaults": func(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{UpdateDefaults: map[string]any{"colour": "grey"}}) + + _, created := post(t, s.CollectionURL(), map[string]any{"key": "k", "colour": "blue"}) + id, _ := created["id"].(string) + + // Omitted on update: not preserved, not cleared -- reset to the + // update-path constant, which need not be any default create ever + // applied. + status, updated := put(t, s.ItemURL(id), map[string]any{"key": "k"}) + if status != http.StatusOK { + t.Fatalf("update = %d", status) + } + if updated["colour"] != "grey" { + t.Errorf("colour = %v, want the update-path grey rather than the stored blue", + updated["colour"]) + } + + // Sending the field still wins the usual way. + status, updated = put(t, s.ItemURL(id), map[string]any{"key": "k", "colour": "red"}) + if status != http.StatusOK || updated["colour"] != "red" { + t.Errorf("a sent value must beat the update default: %d %v", status, updated) + } + }, +} diff --git a/internal/quirkserver/handlers.go b/internal/quirkserver/handlers.go new file mode 100644 index 0000000..44935c7 --- /dev/null +++ b/internal/quirkserver/handlers.go @@ -0,0 +1,296 @@ +package quirkserver + +import ( + "fmt" + "net/http" + "sort" + "strconv" + "strings" +) + +func (s *Server) list(w http.ResponseWriter, r *http.Request) { + if !s.quirks.IgnoresUnknownQueryParams { + if bad := s.badQueryParam(r); bad != "" { + s.fail(w, http.StatusBadRequest, "unknown query parameter", bad) + return + } + } + if bad := s.badTypedParam(r); bad != "" { + s.fail(w, http.StatusBadRequest, "invalid value for query parameter", bad) + return + } + + s.mu.Lock() + ids := make([]string, 0, len(s.objects)) + for id := range s.objects { + ids = append(ids, id) + } + s.mu.Unlock() + + sort.Strings(ids) + + items := make([]any, 0, len(ids)) + for _, id := range ids { + items = append(items, s.project(id, r)) + } + + writeJSON(w, http.StatusOK, map[string]any{envelopeKey: items}) +} + +func (s *Server) create(w http.ResponseWriter, r *http.Request) { + body, err := readJSON(r) + if err != nil { + s.fail(w, http.StatusBadRequest, "malformed body", "") + return + } + + if missing := s.missingRequired(body); missing != "" { + // Naming the field is what lets the auditor write the requirement down + // as observed rather than guessed: an error that does not name it + // could have been about anything. + s.fail(w, http.StatusBadRequest, "missing required field", missing) + return + } + + if field, value := s.rejectedEnumValue(body); field != "" { + s.fail(w, http.StatusBadRequest, "invalid value for "+field, fmt.Sprint(value)) + return + } + + s.mu.Lock() + + id := strconv.Itoa(s.nextID) + s.nextID++ + + obj := map[string]any{"id": id} + for k, v := range body { + if contains(s.quirks.SilentlyDiscards, k) { + // Accepted and thrown away. The response says 201 and the object + // never has it. + continue + } + if contains(s.quirks.NullsInWriteResponse, k) { + // Accepted, stored nowhere, and answered as explicit null by + // project. + continue + } + if dw := s.quirks.DiscardsWhen; dw != nil && k == dw.Then && + fmt.Sprint(body[dw.WhenField]) == fmt.Sprint(dw.WhenValue) { + // The conditional variant: dropped on this branch, stored on every + // other. + continue + } + if sw := s.quirks.SuppressWhenSibling; sw != nil && k == sw.Then && + fmt.Sprint(body[sw.WhenField]) == fmt.Sprint(sw.WhenValue) { + // The field interaction: fine alone, stripped when the sibling + // rides along. + continue + } + if forced, ok := s.quirks.Forces[k]; ok { + // Whatever was sent, the server's own value wins. + obj[k] = forced + continue + } + obj[k] = s.normalise(k, v) + } + + s.applyDefaults(obj, body) + s.applySideEffects(obj, body) + + s.objects[id] = obj + s.mu.Unlock() + + writeJSON(w, http.StatusCreated, s.project(id, r)) +} + +func (s *Server) read(w http.ResponseWriter, r *http.Request, path string) { + id := strings.TrimPrefix(path, itemPrefix) + + s.mu.Lock() + _, exists := s.objects[id] + s.reads[id]++ + seen := s.reads[id] + s.mu.Unlock() + + // Eventual consistency: the object exists but the first N reads deny it. + if exists && seen <= s.quirks.EventuallyConsistentReads { + s.notFound(w) + return + } + + if !exists { + s.notFound(w) + return + } + + writeJSON(w, http.StatusOK, s.project(id, r)) +} + +func (s *Server) update(w http.ResponseWriter, r *http.Request, path string) { + id := strings.TrimPrefix(path, itemPrefix) + + body, err := readJSON(r) + if err != nil { + s.fail(w, http.StatusBadRequest, "malformed body", "") + return + } + + s.mu.Lock() + current, exists := s.objects[id] + s.mu.Unlock() + + if !exists { + s.notFound(w) + return + } + + // Checked before immutability, deliberately: this is the quirk that makes + // a naive audit read a 4xx as immutability when the request shape was + // simply wrong. + if f := s.quirks.RequiresExtraFieldOnUpdate; f != "" { + if _, ok := body[f]; !ok { + s.fail(w, http.StatusBadRequest, "missing required field", f) + return + } + } + + for _, field := range s.quirks.ImmutableAfterCreate { + sent, ok := body[field] + if !ok { + continue + } + if !equalJSON(sent, current[field]) { + s.fail(w, http.StatusBadRequest, "field cannot be modified after creation", field) + return + } + } + + s.mu.Lock() + + updated := map[string]any{"id": id} + if !s.quirks.PutClearsOmitted { + // Everything not mentioned survives. + for k, v := range current { + updated[k] = v + } + } + for k, v := range body { + if contains(s.quirks.SilentlyDiscards, k) { + continue + } + if contains(s.quirks.NullsInWriteResponse, k) { + continue + } + // Accepted, answered 2xx, and not applied. Distinct from + // SilentlyDiscards, which never stores the field at all: here a create + // stored it and the update is the thing that quietly does nothing. + if contains(s.quirks.SilentlyDiscardsOnUpdate, k) { + continue + } + if sw := s.quirks.SuppressWhenSibling; sw != nil && k == sw.Then && + fmt.Sprint(body[sw.WhenField]) == fmt.Sprint(sw.WhenValue) { + // The interaction holds on update too: carrying the create-stored + // value through would hide the suppression from a PUT-based audit. + delete(updated, k) + continue + } + if forced, ok := s.quirks.Forces[k]; ok { + updated[k] = forced + continue + } + updated[k] = s.normalise(k, v) + } + + // An omitted field reverts to the update-path constant, whatever the + // object held -- after the body loop, so sending the field still wins the + // usual way. + for k, v := range s.quirks.UpdateDefaults { + if _, sent := body[k]; !sent { + updated[k] = v + } + } + + s.applySideEffects(updated, body) + + s.objects[id] = updated + s.mu.Unlock() + + writeJSON(w, http.StatusOK, s.project(id, r)) +} + +func (s *Server) delete(w http.ResponseWriter, r *http.Request, path string) { + id := strings.TrimPrefix(path, itemPrefix) + + s.mu.Lock() + s.deletes++ + attempt := s.deletes + s.mu.Unlock() + + if s.quirks.DeleteFails { + s.fail(w, http.StatusInternalServerError, "delete failed", id) + return + } + if n := s.quirks.DeleteFlakyEvery; n > 0 && attempt%n == 0 { + s.fail(w, http.StatusInternalServerError, "delete failed", id) + return + } + + s.mu.Lock() + _, exists := s.objects[id] + delete(s.objects, id) + s.mu.Unlock() + + if !exists { + s.notFound(w) + return + } + + w.WriteHeader(http.StatusNoContent) +} + +// project renders an object as the API would return it, applying the quirks +// that affect what a read sees. +func (s *Server) project(id string, r *http.Request) map[string]any { + s.mu.Lock() + defer s.mu.Unlock() + + obj := s.objects[id] + out := make(map[string]any, len(obj)) + + for k, v := range obj { + // Expansion-gated fields are withheld unless asked for. + if want, gated := s.quirks.ExpansionGated[k]; gated { + if !expansionRequested(r, want) { + continue + } + } + out[k] = v + } + + // Volatile fields differ on every read, which is what makes a perpetual + // diff. + for _, field := range s.quirks.VolatileFields { + s.counter++ + out[field] = fmt.Sprintf("v%d", s.counter) + } + + // Explicit null, everywhere the object is rendered: present-and-null is + // the observation this quirk exists to produce. + for _, field := range s.quirks.NullsInWriteResponse { + out[field] = nil + } + + return out +} + +func expansionRequested(r *http.Request, want string) bool { + if r == nil { + return false + } + for _, v := range r.URL.Query()["expand"] { + if v == want { + return true + } + } + return false +} diff --git a/internal/quirkserver/quirks.go b/internal/quirkserver/quirks.go new file mode 100644 index 0000000..23a78a1 --- /dev/null +++ b/internal/quirkserver/quirks.go @@ -0,0 +1,220 @@ +package quirkserver + +// Conditional describes a requirement that depends on another field's value. +// +// The archetypal case is a port field that matters only when a protocol field +// says tcp. It is the quirk that proves one-field-at-a-time omission reports +// half a truth, and that the auditor must flag the disagreement between two +// fixtures rather than pick a side. +type Conditional struct { + // When is the field/value pair that triggers the requirement. + WhenField string + WhenValue any + // Then is the field that becomes required. + Then string +} + +// Quirks turns each misbehaviour on. +// +// The zero value is a well-behaved API, so a test enables exactly the quirk it +// is about. +type Quirks struct { + // SilentlyDiscards accepts these fields on write and never stores them. + // + // Ground truth for a field that cannot be written, and the trap that makes + // a naive read-back check wrong: the field was demonstrably sent and is + // demonstrably absent. + SilentlyDiscards []string + + // ImmutableAfterCreate refuses to change these fields on update, with a + // problem+json body that names the field. + ImmutableAfterCreate []string + + // RequiresExtraFieldOnUpdate makes update reject any request omitting this + // field, even though create does not need it. + // + // The quirk that proves the immutability protocol's control request is + // load-bearing rather than ceremony: without the control, the auditor sees + // a 4xx on update and concludes immutability when the request shape was + // simply wrong. + RequiresExtraFieldOnUpdate string + + // ConstantDefaults are values assigned when a field is omitted. Ground + // truth for a real server-side default. + ConstantDefaults map[string]any + + // DerivedDefaults assigns a value computed from another field. + // + // An auditor without the derivation check confidently writes this down as + // a static default, which is then a permanent lie. The map is target field + // to source field. + DerivedDefaults map[string]string + + // CounterDefault assigns an incrementing value to this field when omitted, + // so two byte-identical creates differ. Exercises the + // two-identical-creates check. + CounterDefault string + + // RequiredButUndeclared rejects a create omitting these, naming them in + // the error. The key/object_type case. + RequiredButUndeclared []string + + // ConditionallyRequired makes one field required only when another has a + // given value. + ConditionallyRequired *Conditional + + // DiscardsWhen accepts field Then on create and stores it only when the + // request's WhenField does NOT hold WhenValue; on the matching branch it + // is silently dropped. + // + // The conditional variant of SilentlyDiscards, and the matchType case + // exactly: sent on a static tag, 201, and the value is gone; sent on a + // dynamic one, stored and returned. An audit measuring one branch reports + // a half-truth about both. + DiscardsWhen *Conditional + + // WriteSideEffects sets one field as a consequence of another being + // written. + // + // Enabling one measurement silently enabling another: the class of quirk a + // human would never guess from a specification and an auditor genuinely + // can find. Maps the trigger field to the field it also sets. + WriteSideEffects map[string]string + + // NormalisesCase lower-cases these fields' values. + NormalisesCase []string + // TrimsWhitespace strips surrounding whitespace from these fields. + TrimsWhitespace []string + // SortsLists sorts these list-valued fields. + // + // Hand-written providers carry runtime helpers that re-sort collections + // purely to suppress the drift this causes, which is the wrong layer to + // fix it at. + SortsLists []string + + // ExpansionGated returns these fields only when the matching expansion + // query parameter is present. + // + // An audit that reads back once concludes the field is never returned, and + // the generated state mapper then blanks a real value on every refresh. + // Any API with an expand, include or fields parameter can do this. Maps + // field name to the value that reveals it. + ExpansionGated map[string]string + + // SilentlyDiscardsOnUpdate accepts these fields on update, answers 2xx, + // and leaves the stored value alone. Create stores them normally. + // + // The behaviour that has to be distinguishable from immutability, and + // conflating the two is the classic error: an API that refuses a change + // says so, and one that drops it does not. Only the second produces a + // perpetual diff in a generated provider. + SilentlyDiscardsOnUpdate []string + + // PutClearsOmitted makes update replace what is stored rather than + // preserve what the request omits. + PutClearsOmitted bool + + // EventuallyConsistentReads makes the first N reads of a new object 404. + EventuallyConsistentReads int + + // ErrorEnvelope selects which shape errors take. Defaults to problem+json. + ErrorEnvelope Envelope + + // ClosedEnum rejects values outside the listed set, per field. + ClosedEnum map[string][]string + // RejectsValueUnless refuses one field's value except while another field + // holds a given value, keyed "field=value" to a Conditional whose Then is + // unused. + // + // The endpoint-agent case as ground truth: a documented objectType that is + // refused while creating a static tag and perfectly good for a dynamic + // one. An auditor that injects candidates into one arbitrary body writes + // the combination's refusal down as a truth about the value; escalation + // into the other declared fixtures is what corrects it. + RejectsValueUnless map[string]Conditional + + // RejectsDocumentedValue refuses a value the specification documents, per + // field. + // + // The valuable enum result: the specification is stale, and a spec-derived + // validator would have been actively harmful. + RejectsDocumentedValue map[string]string + + // DeleteFails makes every delete fail, which must stop the run creating + // more and leave orphans reported. + DeleteFails bool + // DeleteFlakyEvery makes every Nth delete fail. + DeleteFlakyEvery int + + // RateLimitHeaders emits a rate-limit header trio and 429s once the budget + // is spent, which lets pacing be tested with no live tenant. + RateLimitHeaders bool + // RateLimit is the budget when RateLimitHeaders is set. + RateLimit int + + // VolatileFields change on every read. The modifiedDate perpetual-diff + // class. + VolatileFields []string + + // IgnoresUnknownQueryParams returns 200 for an unrecognised query + // parameter rather than 400. + // + // Calibrates every audit check that depends on unknown *body* fields being + // ignored rather than rejected. + IgnoresUnknownQueryParams bool + + // TypedQueryParams reject a bad value, which is how the error-envelope + // check provokes an error without mutating anything. + TypedQueryParams []string + + // BasePath serves the collection under a prefix, e.g. "/v7". + // + // Real endpoints carry one, and it is the difference between the relative + // path an audit assembles and the full path the wire actually sees. Bugs + // hide behind its absence -- anything comparing the two silently assumes + // they are the same string -- so the fixture is able to model it + // deliberately. + BasePath string + + // NotFoundStatus overrides the status for an absent object. An API that + // returns 403 for another tenant's identifier is indistinguishable from + // one that returns it for an absent one, and that is itself worth + // observing. + NotFoundStatus int + + // Forces stores this value for a field regardless of what was sent, on + // create and update alike, and echoes the forced value back. + // + // The networkMeasurements case: send false, the API stores true, and the + // practitioner's value can never take effect. Distinct from a + // normalisation (a transform of the sent value) and from SilentlyDiscards + // (nothing stored at all). + Forces map[string]any + + // NullsInWriteResponse accepts these fields, stores nothing, and answers + // explicit null for them in every response -- write responses and reads + // alike. + // + // The includeHeaders case, and distinct from SilentlyDiscards on exactly + // the axis the auditor must keep: present-and-null is a different + // observation from absent, and the two must not be conflated. + NullsInWriteResponse []string + + // SuppressWhenSibling drops field Then from storage whenever the same + // request's WhenField holds WhenValue, on create and update alike. + // + // The postBody case: stored and echoed by a small body, stripped the + // moment requestMethod:"get" rides along. Distinct from DiscardsWhen, + // which models the create-branch variant only -- this is the field + // interaction a maximal update meets and a minimal one never does. + SuppressWhenSibling *Conditional + + // UpdateDefaults assigns this value when the field is omitted from an + // update, overriding whatever the object held. Create is untouched. + // + // The full-replace footgun as ground truth: an API whose PUT does not + // preserve an omitted field but resets it to a constant -- and the + // constant may differ from the create-path default, which is why it is its + // own map rather than a flag on ConstantDefaults. + UpdateDefaults map[string]any +} diff --git a/internal/quirkserver/quirkserver.go b/internal/quirkserver/quirkserver.go new file mode 100644 index 0000000..f605719 --- /dev/null +++ b/internal/quirkserver/quirkserver.go @@ -0,0 +1,192 @@ +// Package quirkserver is an API that misbehaves on purpose. +// +// It exists so that every claim the auditor makes is checked against a server +// whose behaviour is known by construction. That is a different and much +// stronger thing than exercising the auditor against a real API: a live tenant +// tells you what *that* API does, while a quirk server tells you whether the +// audit would notice if it did something else. It is the only offline target +// the audit engine is tested against, and it stands in for the live API when +// the whole pipeline is exercised without credentials. +// +// Each switch on Quirks encodes one behaviour observed in a real API. Most +// were drawn from the hardcoded special cases that accumulate in hand-written +// providers -- fixup tables, runtime re-normalisers, lists of fields to treat +// specially. Those are a catalogue of quirks discovered the hard way, one +// production bug at a time, and turning them into switches is what lets an +// audit be *validated* rather than merely run. +// +// Every quirk is asserted to be exhibited by +// TestUnit_Quirkserver_EachQuirkIsExhibited. A switch that silently stopped +// working would make the audit tests that depend on it pass for the wrong +// reason -- which is the one failure mode a ground-truth fixture must not +// have. +package quirkserver + +import ( + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" +) + +// Server is a running quirk server. +type Server struct { + *httptest.Server + + quirks Quirks + + mu sync.Mutex + objects map[string]map[string]any + nextID int + counter int + // reads counts reads per object, for the eventual-consistency quirk. + reads map[string]int + // deletes counts delete attempts, for the flaky-delete quirk. + deletes int + // requests counts everything, so a test can assert an audit's real cost. + requests int + // spent tracks the rate-limit budget. + spent int +} + +// collectionPath and itemPrefix are the paths this fixture serves. +// +// Arbitrary: the auditor is driven by path templates it is handed, so the +// fixture only has to be self-consistent. "/things" rather than any real +// resource name, to keep the fixture from reading as a claim about a +// particular API. +const ( + collectionPath = "/things" + itemPrefix = "/things/" + // envelopeKey is the key a list response wraps its items under, which the + // auditor discovers rather than assumes. + envelopeKey = "things" +) + +// New starts a quirk server. It is closed when the test finishes. +func New(t interface { + Cleanup(func()) + Helper() +}, q Quirks, +) *Server { + t.Helper() + + s := &Server{ + quirks: q, + objects: map[string]map[string]any{}, + nextID: 1, + reads: map[string]int{}, + } + + s.Server = httptest.NewServer(http.HandlerFunc(s.handle)) + t.Cleanup(s.Close) + + return s +} + +// BaseURL is the root a session should be pointed at, including any configured +// prefix. +func (s *Server) BaseURL() string { return s.URL + s.quirks.BasePath } + +// CollectionURL and ItemURL are the addresses an audit is pointed at. +func (s *Server) CollectionURL() string { return s.BaseURL() + collectionPath } +func (s *Server) ItemURL(id string) string { return s.BaseURL() + itemPrefix + id } + +// Requests returns how many requests were served, so a test can assert an +// audit's real cost against the worst case it declared. +func (s *Server) Requests() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.requests +} + +// Objects returns a snapshot of what exists, for asserting cleanup. +func (s *Server) Objects() map[string]map[string]any { + s.mu.Lock() + defer s.mu.Unlock() + + out := make(map[string]map[string]any, len(s.objects)) + for id, obj := range s.objects { + copied := make(map[string]any, len(obj)) + for k, v := range obj { + copied[k] = v + } + out[id] = copied + } + + return out +} + +// Seed inserts an object directly, for tests that need something to read. +func (s *Server) Seed(fields map[string]any) string { + s.mu.Lock() + defer s.mu.Unlock() + + id := strconv.Itoa(s.nextID) + s.nextID++ + + obj := map[string]any{"id": id} + for k, v := range fields { + obj[k] = v + } + s.objects[id] = obj + + return id +} + +func (s *Server) handle(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.requests++ + s.mu.Unlock() + + if s.quirks.RateLimitHeaders { + if done := s.applyRateLimit(w); done { + return + } + } + + // The prefix is stripped once, here, so every handler below matches on the + // same paths whether or not one is configured. + path := strings.TrimPrefix(r.URL.Path, s.quirks.BasePath) + + switch { + case path == collectionPath && r.Method == http.MethodGet: + s.list(w, r) + case path == collectionPath && r.Method == http.MethodPost: + s.create(w, r) + case strings.HasPrefix(path, itemPrefix) && r.Method == http.MethodGet: + s.read(w, r, path) + case strings.HasPrefix(path, itemPrefix) && + (r.Method == http.MethodPut || r.Method == http.MethodPatch): + s.update(w, r, path) + case strings.HasPrefix(path, itemPrefix) && r.Method == http.MethodDelete: + s.delete(w, r, path) + default: + s.fail(w, http.StatusMethodNotAllowed, "unsupported", "") + } +} + +func (s *Server) applyRateLimit(w http.ResponseWriter) bool { + s.mu.Lock() + defer s.mu.Unlock() + + limit := s.quirks.RateLimit + if limit <= 0 { + limit = 240 + } + + s.spent++ + + w.Header().Set("x-organization-rate-limit-limit", strconv.Itoa(limit)) + w.Header().Set("x-organization-rate-limit-remaining", strconv.Itoa(max(0, limit-s.spent))) + w.Header().Set("x-organization-rate-limit-reset", "60") + + if s.spent > limit { + w.Header().Set("retry-after", "60") + w.WriteHeader(http.StatusTooManyRequests) + return true + } + + return false +} diff --git a/internal/quirkserver/quirkserver_test.go b/internal/quirkserver/quirkserver_test.go new file mode 100644 index 0000000..a2f53da --- /dev/null +++ b/internal/quirkserver/quirkserver_test.go @@ -0,0 +1,226 @@ +package quirkserver + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "reflect" + "strings" + "testing" +) + +// client is a plain HTTP client; these tests are about the server. +func get(t *testing.T, url string) (int, map[string]any) { + t.Helper() + return do(t, http.MethodGet, url, nil) +} + +func post(t *testing.T, url string, body map[string]any) (int, map[string]any) { + t.Helper() + return do(t, http.MethodPost, url, body) +} + +func put(t *testing.T, url string, body map[string]any) (int, map[string]any) { + t.Helper() + return do(t, http.MethodPut, url, body) +} + +func do(t *testing.T, method, url string, body map[string]any) (int, map[string]any) { + t.Helper() + + var r io.Reader + if body != nil { + encoded, err := json.Marshal(body) + if err != nil { + t.Fatalf("Marshal: %v", err) + } + r = bytes.NewReader(encoded) + } + + req, err := http.NewRequest(method, url, r) //nolint:noctx // a test fixture + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, url, err) + } + defer func() { _ = resp.Body.Close() }() + + raw, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("reading the body: %v", err) + } + + var out map[string]any + if len(bytes.TrimSpace(raw)) > 0 { + if err := json.Unmarshal(raw, &out); err != nil { + // Not JSON, which is itself something a test may be asserting. + out = map[string]any{"_raw": string(raw)} + } + } + + return resp.StatusCode, out +} + +// TestUnit_Quirkserver_EachQuirkIsExhibited asserts that every switch actually +// misbehaves. This matters more than it looks: a quirk that silently stopped +// working would make every audit test depending on it pass for the wrong +// reason, and a ground-truth fixture that lies is worse than no fixture at +// all. +// +// The exhibit registries are keyed by Quirks field name and the driver walks +// the struct by reflection, so adding a quirk without an exhibit -- or an +// exhibit for a quirk that no longer exists -- fails here rather than rotting +// quietly. Each exhibit asserts the *observable* consequence rather than an +// internal flag. +func TestUnit_Quirkserver_EachQuirkIsExhibited(t *testing.T) { + t.Parallel() + + exhibits := make(map[string]func(*testing.T), len(writeExhibits)+len(readExhibits)) + for name, fn := range writeExhibits { + exhibits[name] = fn + } + for name, fn := range readExhibits { + if _, dup := exhibits[name]; dup { + t.Fatalf("quirk %s has two exhibits; each belongs in exactly one registry", name) + } + exhibits[name] = fn + } + + quirks := reflect.TypeOf(Quirks{}) + for i := range quirks.NumField() { + name := quirks.Field(i).Name + fn, ok := exhibits[name] + if !ok { + t.Errorf("quirk %s has no exhibit -- a switch nobody checks can silently stop working", name) + continue + } + delete(exhibits, name) + t.Run(name, fn) + } + + for name := range exhibits { + t.Errorf("exhibit %s matches no field of Quirks -- the quirk it asserted is gone", name) + } +} + +// TestUnit_Quirkserver_ZeroValueIsWellBehaved: a test enables exactly the +// quirk it is about, so the default has to be an ordinary API. Otherwise every +// audit test would be exercising misbehaviour it did not ask for. +func TestUnit_Quirkserver_ZeroValueIsWellBehaved(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{}) + + status, created := post(t, s.CollectionURL(), map[string]any{"key": "k", "value": "v"}) + if status != http.StatusCreated { + t.Fatalf("create status = %d, want 201", status) + } + id, _ := created["id"].(string) + if id == "" { + t.Fatalf("no id was assigned: %v", created) + } + + status, read := get(t, s.ItemURL(id)) + if status != http.StatusOK { + t.Errorf("read status = %d, want 200", status) + } + // Everything sent comes back unchanged, twice. + if read["key"] != "k" || read["value"] != "v" { + t.Errorf("a well-behaved server should echo what it was sent: %v", read) + } + + _, again := get(t, s.ItemURL(id)) + if read["key"] != again["key"] { + t.Error("a well-behaved server should be stable across reads") + } + + status, listed := get(t, s.CollectionURL()) + if status != http.StatusOK { + t.Errorf("list status = %d, want 200", status) + } + if items, _ := listed["things"].([]any); len(items) != 1 { + t.Errorf("the list should hold one object: %v", listed) + } + + if status, _ := do(t, http.MethodDelete, s.ItemURL(id), nil); status != http.StatusNoContent { + t.Errorf("delete status = %d, want 204", status) + } + if len(s.Objects()) != 0 { + t.Errorf("delete should remove the object: %v", s.Objects()) + } + + // And a second delete reports absence rather than succeeding again. + if status, _ := do(t, http.MethodDelete, s.ItemURL(id), nil); status != http.StatusNotFound { + t.Errorf("deleting twice should 404, got %d", status) + } +} + +func TestUnit_Quirkserver_CountsRequests(t *testing.T) { + t.Parallel() + + // So a test can assert an audit's real cost against the worst case it + // declared -- a request budget that drifts from reality makes every pacing + // decision wrong. + s := New(t, Quirks{}) + + for range 3 { + get(t, s.CollectionURL()) + } + + if got := s.Requests(); got != 3 { + t.Errorf("Requests = %d, want 3", got) + } +} + +func TestUnit_Quirkserver_RejectsMalformedAndUnsupported(t *testing.T) { + t.Parallel() + + s := New(t, Quirks{}) + + req, err := http.NewRequest(http.MethodPost, s.CollectionURL(), strings.NewReader("{not json")) //nolint:noctx // a test fixture + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("Do: %v", err) + } + _ = resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("a malformed body should give 400, got %d", resp.StatusCode) + } + + // A malformed update body, same contract. + upReq, err := http.NewRequest(http.MethodPut, s.ItemURL("1"), strings.NewReader("{not json")) //nolint:noctx // a test fixture + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + upResp, err := http.DefaultClient.Do(upReq) + if err != nil { + t.Fatalf("Do: %v", err) + } + _ = upResp.Body.Close() + if upResp.StatusCode != http.StatusBadRequest { + t.Errorf("a malformed update body should give 400, got %d", upResp.StatusCode) + } + + // An unsupported method on the collection. + if status, _ := do(t, http.MethodDelete, s.CollectionURL(), nil); status != http.StatusMethodNotAllowed { + t.Errorf("status = %d, want 405", status) + } + + // Updating something absent. + if status, _ := put(t, s.ItemURL("absent"), map[string]any{"key": "k"}); status != http.StatusNotFound { + t.Errorf("status = %d, want 404", status) + } +}