From cdff83c055bfd95e82aaf8d1e3c74466d99bf92e Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Thu, 13 Aug 2026 12:13:41 +1000 Subject: [PATCH 1/2] feat(search): match Elasticsearch token semantics exactly (PPT-2644) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-word queries are now OR-joined whole-word matches with a prefix on the final token only — precisely what the old simple_query_string plus Neuroplastic's trailing * produced — replacing the AND-of-prefixes the initial migration shipped. Email-shaped input stays a single quoted whole-address lexeme so user lookup keeps the precision the old field-scoped phrase match provided (OR-splitting an address would match everyone on its domain). Adds the previously-missing dedicated sanitizer spec (9 cases incl. the never-500 vectors). Co-Authored-By: Claude Fable 5 --- spec/text_search_spec.cr | 59 +++++++++++++++++++ src/placeos-rest-api/utilities/text-search.cr | 42 ++++++++++--- 2 files changed, 93 insertions(+), 8 deletions(-) create mode 100644 spec/text_search_spec.cr diff --git a/spec/text_search_spec.cr b/spec/text_search_spec.cr new file mode 100644 index 00000000..e080bc22 --- /dev/null +++ b/spec/text_search_spec.cr @@ -0,0 +1,59 @@ +require "./helper" + +module PlaceOS::Api + describe Utils::TextSearch do + describe ".tsquery" do + it "treats nil, blank and wildcard input as match-all" do + Utils::TextSearch.tsquery(nil).should be_nil + Utils::TextSearch.tsquery("").should be_nil + Utils::TextSearch.tsquery(" ").should be_nil + Utils::TextSearch.tsquery("*").should be_nil + Utils::TextSearch.tsquery("& ! | ( ) ~ \" \\").should be_nil + end + + it "prefix-matches a single token" do + Utils::TextSearch.tsquery("sydney").should eq "sydney:*" + end + + it "OR-joins tokens, prefixing only the final one (Elasticsearch parity)" do + # the old simple_query_string OR-ed terms and Neuroplastic appended + # `*` to the query's last token only + Utils::TextSearch.tsquery("sydney room").should eq "sydney | room:*" + Utils::TextSearch.tsquery("main boardroom 4").should eq "main | boardroom | 4:*" + end + + it "degrades ES field syntax into plain terms (Backoffice zone tag filter)" do + Utils::TextSearch.tsquery("tags:(+level AND +building)").should eq "level | building:*" + end + + it "neutralises ES operators, quotes and boolean words" do + # `garbage:` reads as a field prefix and is stripped with it + Utils::TextSearch.tsquery(%(name:(+weird AND "syntax) | garbage:* ~)).should eq "weird | syntax:*" + Utils::TextSearch.tsquery("name^2 boost").should eq "name | 2 | boost:*" + end + + it "keeps email addresses as a single quoted lexeme" do + # OR-splitting an address would match every user on the domain; the + # whole-address lexeme preserves the old field-scoped precision + Utils::TextSearch.tsquery("adele@example.onmicrosoft.com").should eq "'adele@example.onmicrosoft.com':*" + Utils::TextSearch.tsquery("meeting adele@example.com notes").should eq "meeting | 'adele@example.com' | notes:*" + end + + it "splits hyphenated identifiers" do + Utils::TextSearch.tsquery("sys-abc123").should eq "sys | abc123:*" + end + + it "keeps unicode terms without folding" do + Utils::TextSearch.tsquery("café").should eq "café:*" + end + + it "caps token count and input length without erroring" do + many = (1..40).join(' ') { |i| "tok#{i}" } + query = Utils::TextSearch.tsquery(many).not_nil! + query.split(" | ").size.should eq Utils::TextSearch::MAX_TOKENS + + Utils::TextSearch.tsquery("x" * 20_000).not_nil!.size.should be <= Utils::TextSearch::MAX_QUERY_CHARS + 2 + end + end + end +end diff --git a/src/placeos-rest-api/utilities/text-search.cr b/src/placeos-rest-api/utilities/text-search.cr index 0e9508bc..9b974da9 100644 --- a/src/placeos-rest-api/utilities/text-search.cr +++ b/src/placeos-rest-api/utilities/text-search.cr @@ -1,18 +1,19 @@ module PlaceOS::Api # PPT-2644: translates the free-form `q` search param into a PostgreSQL # tsquery matched against the generated `search_vector` columns - # (see placeos-models migration 20260806100500000). + # (see placeos-models migration 20260810100500000). # # Guarantees: # - never raises, and the output can never produce a tsquery syntax error: - # emitted tokens contain only letters/digits joined with `:*` and `&` + # emitted tokens contain only letters/digits joined with `:*` and `|` # - Elasticsearch-era query syntax that clients still send (field prefixes # like `tags:(+level AND +building)`, boolean operators, quotes, wildcards) # degrades gracefully into plain terms instead of erroring - # - every token is a prefix match (parity with the trailing `*` the old - # Neuroplastic layer appended to every query); tokens are ANDed, which is - # equal-or-stricter than the old OR and matches what autocomplete UIs - # expect (they intersect results client-side) + # - token semantics mirror the old Elasticsearch simple_query_string + # exactly: tokens are OR-joined whole-word matches, with the FINAL token + # a prefix match (Neuroplastic appended `*` to the query's last token) — + # so a record matches when ANY term matches, and the term being typed + # still autocompletes module Utils::TextSearch extend self @@ -22,6 +23,15 @@ module PlaceOS::Api MAX_QUERY_CHARS = 512 MAX_TOKENS = 16 + # Email addresses are kept as a single (quoted) lexeme rather than being + # split: the search vectors index every address both whole and tokenized, + # and whole-address matching is what preserves the precise user-lookup + # behavior clients had via the old field-scoped email phrase match — + # OR-splitting an address would match everyone on the same domain. + # The character class cannot match `'` or `\`, so the quoted lexeme can + # never break out of the tsquery syntax. + EMAIL = /[\p{L}\p{N}._%+-]+@[\p{L}\p{N}.-]+\.[\p{L}]{2,}/ + # Builds the argument for `to_tsquery('simple', ?)` from user input, or # returns `nil` when the input imposes no text filter (nil / blank / "*" / # nothing searchable) — ES treated those as match-all. @@ -29,17 +39,33 @@ module PlaceOS::Api return nil if q.nil? q = q[0, MAX_QUERY_CHARS] if q.size > MAX_QUERY_CHARS + # protect email addresses from tokenization (see EMAIL above); the + # placeholder is alphanumeric so it survives the splits below + emails = [] of String + text = q.gsub(EMAIL) do |address| + emails << address + " placeosemailtoken#{emails.size - 1}x " + end + # drop `field:` prefixes (Backoffice's zone tag filter sends ES syntax # like `tags:(+level AND +building)`) - text = q.gsub(/[\w.]+\s*:/, ' ') + text = text.gsub(/[\w.]+\s*:/, ' ') tokens = text .split(/[^\p{L}\p{N}]+/, remove_empty: true) .reject { |token| OPERATOR_WORDS.includes?(token.downcase) } + .map { |token| + if token =~ /^placeosemailtoken(\d+)x$/ && (address = emails[$1.to_i]?) + "'#{address}'" + else + token + end + } .first(MAX_TOKENS) return nil if tokens.empty? - tokens.join(" & ") { |token| "#{token}:*" } + last = tokens.size - 1 + tokens.map_with_index { |token, i| i == last ? "#{token}:*" : token }.join(" | ") end end end From 590da21e2a36d49bd19dddeeaeac4c959852f0b1 Mon Sep 17 00:00:00 2001 From: Cameron Reeves Date: Thu, 13 Aug 2026 13:33:07 +1000 Subject: [PATCH 2/2] refactor(search): tokenize in a single ordered pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the email placeholder substitution with one scan that captures addresses, consumes field: prefixes and collects words in position order. Eliminates the placeholder-collision edge Greptile flagged (a literal 'placeosemailtokenNx' alongside an email hijacked the mapping) and the latent email-followed-by-colon interaction — both now spec-pinned. Co-Authored-By: Claude Fable 5 --- spec/text_search_spec.cr | 9 +++++ src/placeos-rest-api/utilities/text-search.cr | 37 ++++++++----------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/spec/text_search_spec.cr b/spec/text_search_spec.cr index e080bc22..2505e27c 100644 --- a/spec/text_search_spec.cr +++ b/spec/text_search_spec.cr @@ -39,6 +39,15 @@ module PlaceOS::Api Utils::TextSearch.tsquery("meeting adele@example.com notes").should eq "meeting | 'adele@example.com' | notes:*" end + it "cannot confuse literal input with the email handling (Greptile P2 on #446)" do + # the single-pass tokenizer has no placeholder namespace to collide with + Utils::TextSearch.tsquery("adele@example.com placeosemailtoken0x") + .should eq "'adele@example.com' | placeosemailtoken0x:*" + # an email immediately followed by a colon is not mistaken for a field prefix + Utils::TextSearch.tsquery("adele@example.com: hello") + .should eq "'adele@example.com' | hello:*" + end + it "splits hyphenated identifiers" do Utils::TextSearch.tsquery("sys-abc123").should eq "sys | abc123:*" end diff --git a/src/placeos-rest-api/utilities/text-search.cr b/src/placeos-rest-api/utilities/text-search.cr index 9b974da9..ee0ab849 100644 --- a/src/placeos-rest-api/utilities/text-search.cr +++ b/src/placeos-rest-api/utilities/text-search.cr @@ -32,6 +32,13 @@ module PlaceOS::Api # never break out of the tsquery syntax. EMAIL = /[\p{L}\p{N}._%+-]+@[\p{L}\p{N}.-]+\.[\p{L}]{2,}/ + # One ordered pass over the input: capture whole email addresses, consume + # `field:` prefixes (Backoffice's zone tag filter sends ES syntax like + # `tags:(+level AND +building)`) without emitting them, and collect plain + # word tokens. A single scan means no intermediate placeholder text, so no + # user-typed input can collide with the email handling. + TOKENIZER = /(#{EMAIL.source})|[\w.]+\s*:|([\p{L}\p{N}]+)/ + # Builds the argument for `to_tsquery('simple', ?)` from user input, or # returns `nil` when the input imposes no text filter (nil / blank / "*" / # nothing searchable) — ES treated those as match-all. @@ -39,30 +46,16 @@ module PlaceOS::Api return nil if q.nil? q = q[0, MAX_QUERY_CHARS] if q.size > MAX_QUERY_CHARS - # protect email addresses from tokenization (see EMAIL above); the - # placeholder is alphanumeric so it survives the splits below - emails = [] of String - text = q.gsub(EMAIL) do |address| - emails << address - " placeosemailtoken#{emails.size - 1}x " + tokens = [] of String + q.scan(TOKENIZER) do |match| + break if tokens.size >= MAX_TOKENS + if address = match[1]? + tokens << "'#{address}'" + elsif word = match[2]? + tokens << word unless OPERATOR_WORDS.includes?(word.downcase) + end end - # drop `field:` prefixes (Backoffice's zone tag filter sends ES syntax - # like `tags:(+level AND +building)`) - text = text.gsub(/[\w.]+\s*:/, ' ') - - tokens = text - .split(/[^\p{L}\p{N}]+/, remove_empty: true) - .reject { |token| OPERATOR_WORDS.includes?(token.downcase) } - .map { |token| - if token =~ /^placeosemailtoken(\d+)x$/ && (address = emails[$1.to_i]?) - "'#{address}'" - else - token - end - } - .first(MAX_TOKENS) - return nil if tokens.empty? last = tokens.size - 1 tokens.map_with_index { |token, i| i == last ? "#{token}:*" : token }.join(" | ")