From 744f16f397259d8e5d264f31fbc9ed79f2218c36 Mon Sep 17 00:00:00 2001 From: Albert Zeyer Date: Sat, 17 Oct 2026 06:02:35 -0700 Subject: [PATCH 1/6] Add experimental native English normalization runtime Signed-off-by: Albert Zeyer --- .../test_native_runtime.py | 12 + .../native_runtime/.clang-format | 2 + .../native_runtime/LICENSE.sacremoses | 21 + .../native_runtime/README.md | 103 ++ .../native_runtime/__init__.py | 3 + .../native_runtime/benchmark.py | 78 ++ .../native_runtime/byte_transducer.cc | 1192 +++++++++++++++++ .../native_runtime/byte_transducer.py | 210 +++ .../native_runtime/export_grammars.py | 33 + .../native_runtime/export_text_tables.py | 52 + .../native_runtime/normalize_stream.cc | 82 ++ .../native_runtime/test_byte_transducer.py | 420 ++++++ .../native_runtime/test_text_processing.py | 295 ++++ .../native_runtime/text_processing.h | 319 +++++ .../native_runtime/text_tables.h | 473 +++++++ 15 files changed, 3295 insertions(+) create mode 100644 tests/nemo_text_processing/test_native_runtime.py create mode 100644 tools/text_processing_deployment/native_runtime/.clang-format create mode 100644 tools/text_processing_deployment/native_runtime/LICENSE.sacremoses create mode 100644 tools/text_processing_deployment/native_runtime/README.md create mode 100644 tools/text_processing_deployment/native_runtime/__init__.py create mode 100644 tools/text_processing_deployment/native_runtime/benchmark.py create mode 100644 tools/text_processing_deployment/native_runtime/byte_transducer.cc create mode 100644 tools/text_processing_deployment/native_runtime/byte_transducer.py create mode 100644 tools/text_processing_deployment/native_runtime/export_grammars.py create mode 100644 tools/text_processing_deployment/native_runtime/export_text_tables.py create mode 100644 tools/text_processing_deployment/native_runtime/normalize_stream.cc create mode 100644 tools/text_processing_deployment/native_runtime/test_byte_transducer.py create mode 100644 tools/text_processing_deployment/native_runtime/test_text_processing.py create mode 100644 tools/text_processing_deployment/native_runtime/text_processing.h create mode 100644 tools/text_processing_deployment/native_runtime/text_tables.h diff --git a/tests/nemo_text_processing/test_native_runtime.py b/tests/nemo_text_processing/test_native_runtime.py new file mode 100644 index 000000000..cb5f112bd --- /dev/null +++ b/tests/nemo_text_processing/test_native_runtime.py @@ -0,0 +1,12 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Expose native runtime regressions through the repository's pytest entry point.""" + +import pytest + +from tools.text_processing_deployment.native_runtime.test_byte_transducer import ByteTransducerTest +from tools.text_processing_deployment.native_runtime.test_text_processing import TextProcessingTest + +pytestmark = pytest.mark.run_only_on("CPU") + +__all__ = ["ByteTransducerTest", "TextProcessingTest"] diff --git a/tools/text_processing_deployment/native_runtime/.clang-format b/tools/text_processing_deployment/native_runtime/.clang-format new file mode 100644 index 000000000..f5e3ec5b1 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/.clang-format @@ -0,0 +1,2 @@ +BasedOnStyle: LLVM +ReflowComments: false diff --git a/tools/text_processing_deployment/native_runtime/LICENSE.sacremoses b/tools/text_processing_deployment/native_runtime/LICENSE.sacremoses new file mode 100644 index 000000000..ccf820205 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/LICENSE.sacremoses @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 alvations + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/text_processing_deployment/native_runtime/README.md b/tools/text_processing_deployment/native_runtime/README.md new file mode 100644 index 000000000..071890b65 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/README.md @@ -0,0 +1,103 @@ +# Experimental native transducer runtime + +Keep Pynini/OpenFst for grammar construction and export byte-transition tables for native execution. +The C++17 evaluator has no OpenFst dependency and does not construct per-input FSTs. +Its complete English entry point includes Unicode whitespace collapse, bracket preprocessing, classification, tagged-text parsing, field permutations, verbalization, the postprocessing FST, Moses detokenization and punctuation restoration. + +This is a development prototype, not a supported backend. +The punctuation options enabled in the reference pipeline are implemented natively. +Compatibility targets the pinned English cached-lookahead reference. +Corpus differential tests and synthetic weighted-graph regressions check exact output; this does not establish compatibility with arbitrary grammars or library versions. +Queue selection includes unweighted LIFO and the reference's cached acyclicity. +The former unweighted counterexample is a passing regression, including cached-property and unused-weighted-branch variants. +Passing finite tests does not prove equivalence on every input. +The current evaluator supports standard tropical weights with byte labels on little-endian hosts; generated symbol labels are rejected. +Resource limits raise errors rather than silently pruning paths. +A runtime handle is mutable and must not be called concurrently. + +## Build and test + +Run from this directory: + +```sh +g++ -std=c++17 -O3 -Wall -Wextra -shared -fPIC byte_transducer.cc -o libtn_byte_transducer.so +g++ -std=c++17 -O3 -Wall -Wextra byte_transducer.cc normalize_stream.cc -o normalize_stream +``` + +Run the tests from the repository root: + +```sh +python -m unittest tools.text_processing_deployment.native_runtime.test_byte_transducer tools.text_processing_deployment.native_runtime.test_text_processing +``` + +The test needs Pynini for its independent OpenFst reference. +Tests check minimum-cost membership and exact outputs separately, including negative arcs, float rounding at merges, lookahead phase merges, projected-output traversal, unsupported cycles, UTF-8, literal brackets, failed verbalization and repeated calls. + +Export a normalizer and benchmark independent written/spoken JSONL references: + +```sh +python export_grammars.py --cache-dir CACHE --output-dir GRAMMARS +python benchmark.py --library ./libtn_byte_transducer.so --grammars GRAMMARS --input REFERENCE.jsonl --report REPORT.json --mismatches DIFFERENCES.jsonl +``` + +Each reference row contains written and spoken string fields. +The JSONL benchmark calls the complete native normalizer but retains Python input/output instrumentation. +The standalone normalize_stream executable reads and writes length-framed UTF-8 records with no Python processing or callbacks. +Invoke it as ./normalize_stream GRAMMARS INPUT.frames OUTPUT.frames; each frame is a four-byte little-endian byte count followed by that many UTF-8 bytes. +The driver fails on malformed input, unsupported text, no accepting path or I/O errors; output before an error is partial. +Source-token decoding, corpus selection and provenance/checkpoint integration are separate from this driver. +Use export_fst from byte_transducer.py to export existing tagger, verbalizer and postprocessor FSTs directly. +NativePipeline loads tagger.bin, verbalizer.bin and postprocessor.bin from a supplied directory. +Its normalize method performs complete English processing; normalize_core exposes the transducer stages for diagnostics. +normalize_batch uses one native call for a list of records, with no Python callbacks during normalization. +Batches use the same framing as the standalone driver, allow at most 64 MiB of input or output, and return no partial result on failure. +ByteTransducer evaluates individual exported graphs and returns both output and path cost. +The C ABI also supports native callers without Python. + +## Execution and compatibility + +The evaluator builds the reachable input/grammar DAG in reusable flat workspaces and settles predecessors in topological order. +Precomputed next-byte sets prune impossible paths. +Forced literal and epsilon chains become output fragments while retaining merge predecessors and the sequence of nonzero floating-point weight additions. +LIFO execution uses the uncompressed graph because compression can change visitation order. +Mode-specialized loops avoid repeated execution-option checks. +Direct composition starts uncompressed to avoid rebuilding the predominantly unweighted English verbalizer lattices. +Classification reproduces prepared-lookahead arc ordering; postprocessing reproduces output projection and epsilon removal before shortest-path selection. +These transformations retain the original graph; they do not replace the linguistic rules with heuristics. + +A single native program should preserve stage boundaries. +Globally minimizing a composed classifier/verbalizer graph would not necessarily reproduce selecting the classifier first and then the first accepted field permutation. +Exact output compatibility with the pinned reference remains required before production use. +Equal-cost alternatives depend on float accumulation and traversal order; a higher-precision accumulator is not a compatible replacement. +Lookahead and output-projection modes reject reachable input-epsilon cycles rather than use an incompatible fallback. +The direct-transducer mode retains a bounded cyclic evaluator; its tests check path cost and membership, not universal tie-order equivalence. +Chain compression accounts for the two possible lookahead copies of a grammar state when preserving merges. + +The complete entry point implements the cased, deterministic English reference with both punctuation options enabled, postprocessing enabled and a 729-permutation split limit. +It collapses Unicode whitespace before and after normalization, as the corpus wrapper does. +NeMo's failed-verbalization early return, including escaped brackets and backslashes, is preserved. +This is not a configurable replacement for every Normalizer option. +The audited dependency set is NeMo text processing 1.2.0, Pynini 2.1.6.post1, Python Unicode 15.0.0 and sacremoses 0.2.0. +Changing grammar exports or dependencies requires fresh differential validation. +TNBYTE02 exports retain cached acyclicity rather than recomputing it, since that cache affects reference queue choice. +Re-export legacy TNBYTE01 grammars before direct-mode unweighted execution; missing queue metadata is an error there. +Malformed UTF-8, NUL, unsupported labels and resource-limit failures are explicit errors, not silent fallbacks. + +Relevant implementation references: + +- [RE2 NFA](https://github.com/google/re2/blob/main/re2/nfa.cc): sparse active states and next-byte filtering. +- [Rust regex-automata](https://docs.rs/regex-automata/latest/regex_automata/): multiple execution strategies, bounded lazy DFA caches and one-pass specialization. +- [Hyperscan](https://github.com/intel/hyperscan) and [VectorScan](https://github.com/VectorCamp/vectorscan): hybrid automata, literal matching and architecture-specific acceleration. +- [PCRE2 JIT](https://www.pcre.org/current/doc/html/pcre2jit.html): compiling repeated matching work into machine code. + +These are architectural references, not interchangeable implementations of weighted text transduction. +No code from these engines is copied here. + +English punctuation processing uses build-time Unicode property tables from export_text_tables.py; regenerate text_tables.h with --output when deliberately changing the reference Python/sacremoses version. +The checked tables identify their source versions, and LICENSE.sacremoses preserves the MIT notice for derived detokenization code and tables. +ASCII decoding has NEON and SSE2 paths with a scalar fallback; define TN_DISABLE_SIMD to compare outputs and timing. +Compare SIMD and scalar execution on both ARM64 and x86-64 when changing text processing. + +This backend is opt-in and does not change Normalizer or Sparrowhawk. +Before enabling a new grammar or dependency version, run differential corpus tests and the repository's applicable NeMo/Sparrowhawk checks. +The reference's prepared-lookahead classifier can choose a different equally weighted output from ordinary Pynini composition; this prototype is not a claim of byte-for-byte equivalence to every Normalizer execution mode. diff --git a/tools/text_processing_deployment/native_runtime/__init__.py b/tools/text_processing_deployment/native_runtime/__init__.py new file mode 100644 index 000000000..7f5ad1f33 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Experimental native execution of exported NeMo byte transducers.""" diff --git a/tools/text_processing_deployment/native_runtime/benchmark.py b/tools/text_processing_deployment/native_runtime/benchmark.py new file mode 100644 index 000000000..debe3cbea --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/benchmark.py @@ -0,0 +1,78 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Measure native normalization against supplied JSONL written/spoken references.""" + +import json +import re +import time +from argparse import ArgumentParser +from collections import Counter +from pathlib import Path + +from byte_transducer import NativePipeline + + +def main() -> None: + parser = ArgumentParser(description=__doc__) + parser.add_argument("--library", type=Path, required=True) + parser.add_argument("--grammars", type=Path, required=True) + parser.add_argument("--input", type=Path, nargs="+", required=True) + parser.add_argument("--report", type=Path, required=True) + parser.add_argument("--mismatches", type=Path, required=True) + args = parser.parse_args() + started = time.perf_counter() + native = NativePipeline(args.library, args.grammars) + setup_seconds = time.perf_counter() - started + counts: Counter[str] = Counter() + seconds: Counter[str] = Counter() + started = time.perf_counter() + try: + with args.mismatches.open("w") as differences: + for path in args.input: + with path.open() as source: + for line in source: + row = json.loads(line) + written = row["written"] + counts["rows"] += 1 + begin = time.perf_counter() + output = native.normalize(written) + seconds["native_normalization"] += time.perf_counter() - begin + if output is None: + counts["no_path"] += 1 + differences.write( + json.dumps({"written": written, "error": "no path"}) + + "\n" + ) + continue + reference = row["spoken"] + if output != reference: + counts["exact_differences"] += 1 + counts["word_token_differences"] += re.findall( + r"\w+", output + ) != re.findall(r"\w+", reference) + differences.write( + json.dumps( + { + "written": written, + "candidate": output, + "reference": reference, + } + ) + + "\n" + ) + report = { + "counts": dict(counts), + "seconds": dict(seconds), + "setup_seconds": setup_seconds, + "elapsed_seconds": time.perf_counter() - started, + "scope": "Includes JSONL input and mismatch output; excludes source-token decoding and corpus output.", + "remaining_python": "JSONL benchmark input/output and call wrapper; use normalize_stream for a native record loop.", + } + args.report.write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2)) + finally: + native.close() + + +if __name__ == "__main__": + main() diff --git a/tools/text_processing_deployment/native_runtime/byte_transducer.cc b/tools/text_processing_deployment/native_runtime/byte_transducer.cc new file mode 100644 index 000000000..78219e319 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/byte_transducer.cc @@ -0,0 +1,1192 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// +// Experimental byte-transducer evaluator. +// Grammar construction remains in Pynini; this file has no OpenFst dependency. +// State merging follows weighted Viterbi semantics, not regexp leftmost-first semantics. +#include "text_processing.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +constexpr float kInf = std::numeric_limits::infinity(); +constexpr uint32_t kNone = UINT32_MAX; +struct Arc { + uint32_t input, output, target; + float weight; +}; +struct State { + uint32_t begin, end; + float final_weight; +}; +struct Trace { + uint32_t parent, output; +}; +struct Tail { + uint32_t target, output; + std::vector weights; + std::string input{}; + bool last_epsilon = false; +}; +struct Frontier { + std::vector costs; + std::vector paths, active; + std::vector queued; + explicit Frontier(size_t n) : costs(n, kInf), paths(n, kNone), queued(n, 0) {} + void clear() { + for (uint32_t s : active) { + costs[s] = kInf; + paths[s] = kNone; + queued[s] = 0; + } + active.clear(); + } +}; + +struct DagNode { + uint32_t state, pos, begin, end, parent = kNone, via = kNone; + float cost = kInf; + uint8_t mark = 0; +}; +struct DagEdge { + uint32_t node, arc; +}; +// Reuse per-handle storage and a flat state/position index across records. +struct DagWorkspace { + std::vector nodes; + std::vector edges; + std::vector stack, order, slots, used; + static size_t hash(uint64_t key) { + key ^= key >> 33; + key *= 0xff51afd7ed558ccdULL; + key ^= key >> 33; + return static_cast(key); + } + void reset() { + for (uint32_t i : used) + slots[i] = kNone; + used.clear(); + nodes.clear(); + edges.clear(); + stack.clear(); + order.clear(); + } + void grow() { + slots.assign(slots.empty() ? 4096 : slots.size() * 2, kNone); + used.clear(); + for (uint32_t i = 0; i < nodes.size(); ++i) { + auto &n = nodes[i]; + uint64_t key = (uint64_t(n.pos) << 32) | n.state; + size_t j = hash(key) & (slots.size() - 1); + while (slots[j] != kNone) + j = (j + 1) & (slots.size() - 1); + slots[j] = i; + used.push_back(j); + } + } + uint32_t get(uint32_t state, uint32_t pos) { + if ((nodes.size() + 1) * 2 >= slots.size()) + grow(); + uint64_t key = (uint64_t(pos) << 32) | state; + size_t j = hash(key) & (slots.size() - 1); + while (slots[j] != kNone) { + uint32_t id = slots[j]; + const auto &n = nodes[id]; + if (n.state == state && n.pos == pos) + return id; + j = (j + 1) & (slots.size() - 1); + } + if (nodes.size() >= 10000000) + throw std::runtime_error("DAG workspace limit"); + uint32_t id = nodes.size(); + nodes.push_back({state, pos, 0, 0}); + slots[j] = id; + used.push_back(j); + return id; + } +}; + +class Program { +public: + std::vector states; + std::vector arcs; + uint32_t start; + bool grammar_zero = true, known_acyclic = false, has_queue_properties = false; + std::vector> next_labels; + std::vector tails, literal_tails; + std::vector fragments; + std::vector traces; + std::vector queue; + Frontier first, second; + std::string result; + size_t relaxations = 0; + + explicit Program(const char *path) : start(0), first(0), second(0) { + std::ifstream f(path, std::ios::binary); + if (!f) + throw std::runtime_error("Cannot open transducer"); + auto read = [&](void *p, size_t n) { + if (!f.read(static_cast(p), n)) + throw std::runtime_error("Truncated transducer"); + }; + const uint32_t endian = 1; + if (*reinterpret_cast(&endian) != 1) + throw std::runtime_error("This prototype requires a little-endian host"); + char magic[8]; + read(magic, 8); + has_queue_properties = std::memcmp(magic, "TNBYTE02", 8) == 0; + if (!has_queue_properties && std::memcmp(magic, "TNBYTE01", 8)) + throw std::runtime_error("Invalid transducer format"); + uint32_t n, m; + read(&n, 4); + read(&m, 4); + read(&start, 4); + if (has_queue_properties) { + uint32_t flags; + read(&flags, 4); + if (flags & ~1u) + throw std::runtime_error("Unsupported queue properties"); + known_acyclic = flags & 1; + } + if (!n || start >= n || n > 10000000 || m > 100000000) + throw std::runtime_error("Invalid transducer dimensions"); + states.resize(n); + arcs.resize(m); + for (auto &s : states) { + read(&s.begin, 4); + read(&s.end, 4); + read(&s.final_weight, 4); + if (s.begin > s.end || s.end > m || std::isnan(s.final_weight) || + s.final_weight == -kInf) + throw std::runtime_error("Invalid state"); + } + for (auto &a : arcs) { + read(&a.input, 4); + read(&a.output, 4); + read(&a.target, 4); + read(&a.weight, 4); + if (a.input > 255 || a.output > 255 || a.target >= n || + !std::isfinite(a.weight)) + throw std::runtime_error("Unsupported arc"); + } + for (const auto &s : states) { + for (uint32_t i = s.begin + (s.begin < s.end); i < s.end; ++i) + if (arcs[i - 1].input > arcs[i].input) + throw std::runtime_error("Unsorted arcs"); + } + for (const auto &arc : arcs) + grammar_zero &= arc.weight == 0; + // Compute possible next bytes through epsilon paths once, rather than + // exploring paths that cannot consume the current byte on every call. + next_labels.resize(n); + std::vector> predecessors(n); + std::deque pending; + std::vector queued(n, 1); + for (uint32_t s = 0; s < n; ++s) { + if (states[s].final_weight != kInf) + next_labels[s][4] = 1; + for (uint32_t j = states[s].begin; j < states[s].end; ++j) { + const auto &a = arcs[j]; + if (!a.input) + predecessors[a.target].push_back(s); + else + next_labels[s][a.input / 64] |= uint64_t{1} << (a.input % 64); + } + pending.push_back(s); + } + while (!pending.empty()) { + uint32_t s = pending.front(); + pending.pop_front(); + queued[s] = 0; + for (uint32_t before : predecessors[s]) { + bool changed = false; + for (size_t k = 0; k < 5; ++k) { + uint64_t combined = next_labels[before][k] | next_labels[s][k]; + changed |= combined != next_labels[before][k]; + next_labels[before][k] = combined; + } + if (changed && !queued[before]) { + pending.push_back(before); + queued[before] = 1; + } + } + } + // Fuse forced epsilon chains into output fragments without changing the + // order of floating-point weight additions. + // A lookahead state can have both an ordinary and a pending-label copy. + // Their outgoing arcs can merge even when the grammar target has one predecessor. + std::vector phases(n, 0); + phases[start] = 1; + for (const auto &a : arcs) + phases[a.target] |= a.input ? 1 : 2; + std::vector indegree(n, 0); + for (uint32_t source = 0; source < n; ++source) + for (uint32_t a = states[source].begin; a < states[source].end; ++a) + indegree[arcs[a].target] += phases[source] == 3 ? 2 : 1; + auto compile_tails = [&](bool include_input) { + std::vector compiled; + compiled.reserve(n); + for (uint32_t s = 0; s < n; ++s) { + Tail tail{s, 0, {}}; + std::string fragment; + size_t steps = 0; + while (steps < 256) { + const auto &st = states[tail.target]; + // Keep merge points: a preference can disappear if a large later cost + // is added before the competing prefix costs are compared. + if (indegree[tail.target] > 1 || st.final_weight != kInf || + st.end - st.begin != 1 || indegree[arcs[st.begin].target] > 1 || + (!include_input && arcs[st.begin].input)) + break; + const auto &a = arcs[st.begin]; + if (a.output) + fragment.push_back(static_cast(a.output)); + if (a.input) + tail.input.push_back(static_cast(a.input)); + tail.last_epsilon = !a.input; + if (a.weight != 0) + tail.weights.push_back(a.weight); + tail.target = a.target; + ++steps; + } + // Leave long chains and cycles to the bounded general evaluator. + if (steps == 256) { + tail = Tail{s, 0, {}}; + fragment.clear(); + } + if (!fragment.empty()) { + tail.output = static_cast(fragments.size() + 256); + fragments.push_back(std::move(fragment)); + } + compiled.push_back(std::move(tail)); + } + return compiled; + }; + tails = compile_tails(false); + literal_tails = compile_tails(true); + first = Frontier(n); + second = Frontier(n); + traces.reserve(65536); + queue.reserve(4096); + } + + bool relax(Frontier &f, uint32_t s, float cost, uint32_t parent, + uint32_t output) { + const auto &tail = tails[s]; + for (float w : tail.weights) + cost += w; + s = tail.target; + if (!(cost < f.costs[s])) + return false; + if (++relaxations > 10000000) + throw std::runtime_error("Relaxation limit reached; possible epsilon " + "cycle or excessive ambiguity"); + if (f.costs[s] == kInf) + f.active.push_back(s); + if (output) { + if (traces.size() >= 10000000) + throw std::runtime_error("Trace memory limit reached"); + traces.push_back({parent, output}); + parent = static_cast(traces.size() - 1); + } + if (tail.output) { + if (traces.size() >= 10000000) + throw std::runtime_error("Trace memory limit reached"); + traces.push_back({parent, tail.output}); + parent = static_cast(traces.size() - 1); + } + f.costs[s] = cost; + f.paths[s] = parent; + return true; + } + + bool can_consume(uint32_t s, uint32_t label) const { + return next_labels[s][label / 64] & (uint64_t{1} << (label % 64)); + } + + void epsilon_closure(Frontier &f, uint32_t label) { + queue.clear(); + for (uint32_t s : f.active) + if (can_consume(s, label)) { + queue.push_back(s); + f.queued[s] = 1; + } + for (size_t p = 0; p < queue.size(); ++p) { + uint32_t s = queue[p]; + f.queued[s] = 0; + const float cost = f.costs[s]; + const uint32_t path = f.paths[s]; + const State &st = states[s]; + for (uint32_t j = st.begin; j < st.end && arcs[j].input == 0; ++j) { + const Arc &a = arcs[j]; + if (can_consume(a.target, label) && + relax(f, a.target, cost + a.weight, path, a.output) && + !f.queued[tails[a.target].target]) { + uint32_t terminal = tails[a.target].target; + queue.push_back(terminal); + f.queued[terminal] = 1; + } + } + } + } + + bool project_output = false; + bool rewrite_projected(float &best_cost, size_t input_size) { + struct PArc { + uint32_t target, output; + float weight; + }; + const auto &nodes = dag.nodes; + size_t n = nodes.size(); + std::vector> outgoing(n); + std::vector final(n, kInf), distance(n, kInf); + std::vector incoming(n, 0), seen(n, 0); + std::vector rank(n), pending, closure; + incoming[0] = 1; + for (uint32_t i = 0; i < dag.order.size(); ++i) + rank[dag.order[i]] = i; + for (uint32_t i = 0; i < n; ++i) { + if (nodes[i].pos == input_size) + final[i] = states[nodes[i].state / 2].final_weight; + for (uint32_t e = nodes[i].begin; e < nodes[i].end; ++e) { + auto edge = dag.edges[e]; + const auto &arc = arcs[edge.arc]; + outgoing[i].push_back({edge.node, arc.output, arc.weight}); + if (arc.output) + incoming[edge.node] = 1; + } + } + // NeMo's top_rewrite projects output labels, removes epsilon paths, then + // selects a path; direct transducer traversal has different tie behavior. + for (uint32_t source : dag.order) { + if (!incoming[source]) + continue; + if (std::none_of(outgoing[source].begin(), outgoing[source].end(), + [](const PArc &arc) { return !arc.output; })) { + std::reverse(outgoing[source].begin(), outgoing[source].end()); + continue; + } + closure.clear(); + pending.clear(); + pending.push_back(source); + while (!pending.empty()) { + uint32_t id = pending.back(); + pending.pop_back(); + if (seen[id]) + continue; + seen[id] = 1; + closure.push_back(id); + for (const auto &arc : outgoing[id]) + if (!arc.output) + pending.push_back(arc.target); + } + for (uint32_t id : closure) { + seen[id] = 0; + distance[id] = kInf; + } + std::sort(closure.begin(), closure.end(), + [&](uint32_t a, uint32_t b) { return rank[a] > rank[b]; }); + distance[source] = 0; + for (uint32_t id : closure) + for (const auto &arc : outgoing[id]) + if (!arc.output) + distance[arc.target] = + std::min(distance[arc.target], distance[id] + arc.weight); + std::vector expanded; + std::unordered_map positions; + pending.push_back(source); + float final_weight = kInf; + while (!pending.empty()) { + uint32_t id = pending.back(); + pending.pop_back(); + if (seen[id]) + continue; + seen[id] = 1; + for (const auto &arc : outgoing[id]) { + if (!arc.output) { + if (!seen[arc.target]) + pending.push_back(arc.target); + continue; + } + uint64_t key = (uint64_t(arc.output) << 32) | arc.target; + auto [it, inserted] = positions.emplace(key, expanded.size()); + float weight = distance[id] + arc.weight; + if (inserted) + expanded.push_back({arc.target, arc.output, weight}); + else + expanded[it->second].weight = + std::min(expanded[it->second].weight, weight); + } + final_weight = std::min(final_weight, distance[id] + final[id]); + } + for (uint32_t id : closure) + seen[id] = 0; + // OpenFst restores the collected arcs by popping from the back. + std::reverse(expanded.begin(), expanded.end()); + outgoing[source] = std::move(expanded); + final[source] = final_weight; + } + // Epsilon removal changes arc order, so recompute the reference DFS order. + std::vector cursor(n, 0), order; + pending.push_back(0); + seen[0] = 1; + while (!pending.empty()) { + uint32_t id = pending.back(); + if (cursor[id] < outgoing[id].size()) { + uint32_t target = outgoing[id][cursor[id]++].target; + if (!seen[target]) { + seen[target] = 1; + pending.push_back(target); + } + } else { + order.push_back(id); + pending.pop_back(); + } + } + std::vector coaccessible(n, 0); + bool unweighted = true; + for (uint32_t id : order) { + coaccessible[id] = final[id] != kInf; + for (const auto &arc : outgoing[id]) { + coaccessible[id] |= coaccessible[arc.target]; + if (coaccessible[arc.target]) + unweighted &= arc.weight == 0; + } + } + // RmEpsilon clears the known acyclic property; ignore dead paths as Connect does. + if (unweighted) { + std::fill(seen.begin(), seen.end(), 0); + pending.clear(); + order.clear(); + pending.push_back(0); + seen[0] = 1; + while (!pending.empty()) { + uint32_t id = pending.back(); + pending.pop_back(); + order.push_back(id); + for (const auto &arc : outgoing[id]) + if (coaccessible[arc.target] && !seen[arc.target]) { + seen[arc.target] = 1; + pending.push_back(arc.target); + } + } + std::reverse(order.begin(), order.end()); + } + std::fill(distance.begin(), distance.end(), kInf); + distance[0] = 0; + std::vector parent(n, kNone), output(n, 0); + best_cost = kInf; + uint32_t last = kNone; + for (auto it = order.rbegin(); it != order.rend(); ++it) { + uint32_t id = *it; + float cost = distance[id] + final[id]; + if (cost < best_cost) { + best_cost = cost; + last = id; + } + for (const auto &arc : outgoing[id]) { + cost = distance[id] + arc.weight; + if (cost < distance[arc.target]) { + distance[arc.target] = cost; + parent[arc.target] = id; + output[arc.target] = arc.output; + } + } + } + result.clear(); + if (last == kNone) + return false; + for (uint32_t id = last; parent[id] != kNone; id = parent[id]) + result.push_back(output[id]); + std::reverse(result.begin(), result.end()); + return true; + } + + DagWorkspace dag; + bool push_prefix = false; + // Specialize record execution so each arc does not retest the same mode flags. + template + bool rewrite_dag_impl(const char *data, size_t size, float &best_cost) { + if (size > UINT32_MAX) + throw std::runtime_error("Input length limit"); + auto &nodes = dag.nodes; + auto &edges = dag.edges; + auto &stack = dag.stack; + auto &order = dag.order; + dag.reset(); + for (size_t i = 0; i < size; ++i) + if (!data[i]) + throw std::runtime_error("Embedded NUL is unsupported"); + dag.get(start * 2, 0); + stack.push_back(0); + while (!stack.empty()) { + uint32_t id = stack.back(); + if (!nodes[id].mark) { + uint32_t state = nodes[id].state / 2, pos = nodes[id].pos; + uint32_t label = + pos < size ? static_cast(data[pos]) : 256; + nodes[id].mark = 1; + nodes[id].begin = edges.size(); + const auto &st = states[state]; + auto add = [&](uint32_t a) { + const auto &arc = arcs[a]; + uint32_t nextpos = pos + (arc.input != 0); + uint32_t nextlabel = + nextpos < size ? static_cast(data[nextpos]) : 256; + if (!can_consume(arc.target, nextlabel)) + return; + const auto &tail = literal_tails[arc.target]; + uint32_t terminal = arc.target; + bool last_epsilon = !arc.input; + if (compress && tail.target != arc.target) { + if (tail.input.size() > size - nextpos || + std::memcmp(data + nextpos, tail.input.data(), + tail.input.size())) + return; + nextpos += tail.input.size(); + terminal = tail.target; + if (terminal != arc.target) + last_epsilon = tail.last_epsilon; + nextlabel = nextpos < size + ? static_cast(data[nextpos]) + : 256; + if (!can_consume(terminal, nextlabel)) + return; + } + bool prefix = lookahead && nextpos < size && last_epsilon; + uint32_t target = dag.get(terminal * 2 + prefix, nextpos); + if (edges.size() >= 10000000) + throw std::runtime_error("DAG edge limit"); + edges.push_back({target, a}); + }; + uint32_t epsilon_end = st.begin; + while (epsilon_end < st.end && !arcs[epsilon_end].input) + ++epsilon_end; + auto add_eps = [&]() { + for (uint32_t a = st.begin; a < epsilon_end; ++a) + add(a); + }; + auto add_match = [&]() { + if (label == 256) + return; + auto it = std::lower_bound( + arcs.begin() + epsilon_end, arcs.begin() + st.end, label, + [](const Arc &arc, uint32_t value) { return arc.input < value; }); + for (; it != arcs.begin() + st.end && it->input == label; ++it) + add(it - arcs.begin()); + }; + // Prepared label lookahead lists the pushed label before epsilon arcs. + // That order determines the selected output when costs tie. + if (nodes[id].state & 1) { + add_match(); + add_eps(); + } else { + add_eps(); + add_match(); + } + nodes[id].end = edges.size(); + } + if (nodes[id].parent == kNone) + nodes[id].parent = nodes[id].begin; + if (nodes[id].parent < nodes[id].end) { + uint32_t child = edges[nodes[id].parent++].node; + if (nodes[child].mark == 1) { + if (lookahead || project_output) + throw std::runtime_error( + "Input-epsilon cycles are unsupported in compatibility modes"); + return rewrite_fast(data, size, best_cost); + } + if (nodes[child].mark == 0) + stack.push_back(child); + } else { + nodes[id].mark = 2; + nodes[id].parent = kNone; + order.push_back(id); + stack.pop_back(); + } + } + if (project_output) + return rewrite_projected(best_cost, size); + // Only direct composition needs Connect pruning to choose its queue. + std::vector coaccessible; + if (!lookahead && !known_acyclic) { + coaccessible.assign(nodes.size(), 0); + for (uint32_t id : order) { + coaccessible[id] = nodes[id].pos == size && + states[nodes[id].state / 2].final_weight != kInf; + for (uint32_t e = nodes[id].begin; e < nodes[id].end; ++e) + coaccessible[id] |= coaccessible[edges[e].node]; + } + } + bool unweighted = grammar_zero; + if (!unweighted && (lookahead || !known_acyclic)) { + unweighted = true; + for (const auto &edge : edges) { + const auto &arc = arcs[edge.arc]; + if (!lookahead && !coaccessible[edge.node]) + continue; + if (arc.weight != 0 || + (compress && !literal_tails[arc.target].weights.empty())) { + unweighted = false; + break; + } + } + } + // AutoQueue tests known acyclicity before falling back to unweighted LIFO. + // Prepared lookahead retains the acyclic property for empty input only. + bool lifo = unweighted && !(lookahead ? size == 0 : known_acyclic); + if (lifo && !lookahead && !has_queue_properties) + throw std::runtime_error( + "Re-export legacy grammar with cached queue properties"); + if (lifo && compress) + return rewrite_dag_impl(data, size, best_cost); + if (lifo) { + std::vector seen(nodes.size(), 0); + stack.clear(); + order.clear(); + stack.push_back(0); + seen[0] = 1; + while (!stack.empty()) { + uint32_t id = stack.back(); + stack.pop_back(); + order.push_back(id); + for (uint32_t e = nodes[id].begin; e < nodes[id].end; ++e) { + uint32_t target = edges[e].node; + if (!lookahead && !coaccessible[target]) + continue; + if (!seen[target]) { + seen[target] = 1; + stack.push_back(target); + } + } + } + std::reverse(order.begin(), order.end()); + } + // Settle every predecessor before propagating a cost or choosing a tie. + nodes[0].cost = 0; + best_cost = kInf; + uint32_t last = kNone; + for (auto it = order.rbegin(); it != order.rend(); ++it) { + uint32_t id = *it; + const auto &node = nodes[id]; + if (node.pos == size) { + float cost = node.cost + states[node.state / 2].final_weight; + if (cost < best_cost) { + best_cost = cost; + last = id; + } + } + for (uint32_t e = node.begin; e < node.end; ++e) { + const auto &edge = edges[e]; + const auto &arc = arcs[edge.arc]; + float cost = node.cost + arc.weight; + if (compress) + for (float weight : literal_tails[arc.target].weights) + cost += weight; + auto &next = nodes[edge.node]; + if (cost < next.cost) { + next.cost = cost; + next.parent = id; + next.via = edge.arc; + } + } + } + result.clear(); + if (last == kNone) + return false; + for (uint32_t id = last; nodes[id].parent != kNone; id = nodes[id].parent) { + const auto &arc = arcs[nodes[id].via]; + const auto &tail = literal_tails[arc.target]; + if (compress && tail.output) { + const auto &fragment = fragments[tail.output - 256]; + result.append(fragment.rbegin(), fragment.rend()); + } + if (arc.output) + result.push_back(arc.output); + } + std::reverse(result.begin(), result.end()); + return true; + } + bool rewrite_dag(const char *data, size_t size, float &cost) { + if (push_prefix) { + if (!grammar_zero && !project_output) + return rewrite_dag_impl(data, size, cost); + return rewrite_dag_impl(data, size, cost); + } + // Verbalizer lattices are usually unweighted; build once without compression. + return rewrite_dag_impl(data, size, cost); + } + bool rewrite(const char *data, size_t size, float &cost) { + return rewrite_dag(data, size, cost); + } + + bool rewrite_fast(const char *data, size_t size, float &best_cost) { + first.clear(); + second.clear(); + traces.clear(); + result.clear(); + relaxations = 0; + Frontier *current = &first; + Frontier *next = &second; + relax(*current, start, 0, kNone, 0); + for (size_t p = 0; p < size; ++p) { + const uint32_t ch = static_cast(data[p]); + epsilon_closure(*current, ch); + // Label zero denotes epsilon, so literal NUL requires a different alphabet. + if (!ch) + throw std::runtime_error("Embedded NUL is unsupported"); + next->clear(); + for (uint32_t s : current->active) { + const State &st = states[s]; + auto a = std::lower_bound( + arcs.begin() + st.begin, arcs.begin() + st.end, ch, + [](const Arc &a, uint32_t label) { return a.input < label; }); + for (; a != arcs.begin() + st.end && a->input == ch; ++a) + relax(*next, a->target, current->costs[s] + a->weight, + current->paths[s], a->output); + } + std::swap(current, next); + if (current->active.empty()) { + best_cost = kInf; + return false; + } + } + epsilon_closure(*current, 256); + best_cost = kInf; + uint32_t path = kNone; + for (uint32_t s : current->active) { + float cost = current->costs[s] + states[s].final_weight; + if (cost < best_cost) { + best_cost = cost; + path = current->paths[s]; + } + } + if (best_cost == kInf) + return false; + for (; path != kNone; path = traces[path].parent) { + uint32_t output = traces[path].output; + if (output < 256) + result.push_back(static_cast(output)); + else { + const auto &fragment = fragments[output - 256]; + result.append(fragment.rbegin(), fragment.rend()); + } + } + std::reverse(result.begin(), result.end()); + return true; + } +}; +thread_local std::string error; +} // namespace + +extern "C" { +void *tn_load(const char *path) { + try { + return new Program(path); + } catch (const std::exception &e) { + error = e.what(); + return nullptr; + } +} +void tn_delete(void *p) { delete static_cast(p); } +const char *tn_error() { return error.c_str(); } +// Result memory belongs to the handle and is valid until its next rewrite. +// One handle is not safe for simultaneous calls. +int tn_rewrite(void *p, const char *text, size_t length, const char **output, + size_t *size, float *cost) { + try { + if (!p || !text || !output || !size || !cost) + throw std::runtime_error("Null argument"); + auto &program = *static_cast(p); + bool found = program.rewrite(text, length, *cost); + *output = program.result.data(); + *size = program.result.size(); + return found ? 0 : 1; + } catch (const std::exception &e) { + error = e.what(); + return -1; + } +} +} + +namespace { +struct Field { + enum Kind { String, Boolean, Object, Missing } kind; + std::string key, value; + std::vector children; +}; + +class TagParser { + const std::string &text; + size_t pos = 0; + void spaces() { + while (pos < text.size() && text[pos] == ' ') + ++pos; + } + void expect(char c) { + if (pos == text.size() || text[pos++] != c) + throw std::runtime_error("Invalid tagged text"); + } + +public: + explicit TagParser(const std::string &input) : text(input) {} + std::vector parse(size_t depth = 0) { + if (depth > 64) + throw std::runtime_error("Tagged nesting limit exceeded"); + std::vector result; + spaces(); + while (pos < text.size() && text[pos] != '}') { + size_t begin = pos; + while (pos < text.size() && + ((text[pos] >= 'a' && text[pos] <= 'z') || + (text[pos] >= 'A' && text[pos] <= 'Z') || text[pos] == '_')) + ++pos; + if (pos == begin) + throw std::runtime_error("Invalid tag key"); + Field f; + f.key = text.substr(begin, pos - begin); + spaces(); + if (f.key == "preserve_order") { + expect(':'); + spaces(); + if (text.compare(pos, 4, "true") != 0) + throw std::runtime_error("Invalid preserve_order"); + pos += 4; + f.kind = Field::Boolean; + } else if (pos < text.size() && text[pos] == '{') { + ++pos; + f.kind = Field::Object; + f.children = parse(depth + 1); + expect('}'); + } else { + expect(':'); + spaces(); + expect('"'); + begin = pos; + while (pos + 1 < text.size() && + !(text[pos] == '"' && text[pos + 1] == ' ')) + ++pos; + if (pos + 1 >= text.size()) + throw std::runtime_error("Unterminated tag string"); + f.kind = pos == begin ? Field::Missing : Field::String; + f.value = text.substr(begin, pos - begin); + ++pos; + } + // NeMo's top-level list preserves repeated "tokens"; nested dictionaries overwrite duplicate keys. + auto old = + depth ? std::find_if(result.begin(), result.end(), + [&](const Field &x) { return x.key == f.key; }) + : result.end(); + if (old != result.end()) + *old = std::move(f); + else + result.push_back(std::move(f)); + spaces(); + } + if (!depth && pos != text.size()) + throw std::runtime_error("Trailing tag text"); + return result; + } +}; + +size_t permutation_count(const std::vector &fields) { + size_t result = 1; + for (size_t i = 2; i <= fields.size(); ++i) { + if (result > 729 / i) + return 730; + result *= i; + } + for (const auto &f : fields) + if (f.kind == Field::Object) { + size_t sub = permutation_count(f.children); + if (result > 729 / sub) + return 730; + result *= sub; + } + return result; +} + +std::vector permutations(const std::vector &fields) { + bool preserve = false; + std::vector order; + std::vector> pieces; + for (size_t i = 0; i < fields.size(); ++i) { + const auto &f = fields[i]; + order.push_back(i); + preserve |= f.key == "preserve_order"; + if (f.kind == Field::Missing) + return {}; + if (f.kind == Field::String) + pieces.push_back({f.key + ": \"" + f.value + "\" "}); + else if (f.kind == Field::Boolean) + pieces.push_back({f.key + ": true "}); + else { + auto values = permutations(f.children); + for (auto &v : values) + v = " " + f.key + " { " + v + " } "; + pieces.push_back(std::move(values)); + } + } + std::vector result; + do { + std::vector partial(1); + for (size_t i : order) { + std::vector expanded; + for (const auto &before : partial) + for (const auto &after : pieces[i]) + expanded.push_back(before + after); + partial = std::move(expanded); + } + result.insert(result.end(), partial.begin(), partial.end()); + if (result.size() > 729) + throw std::runtime_error("Permutation limit exceeded"); + } while (!preserve && std::next_permutation(order.begin(), order.end())); + return result; +} + +std::string collapse_spaces(const std::string &text) { + std::string result; + result.reserve(text.size()); + for (char c : text) { + if (c != ' ' || (!result.empty() && result.back() != ' ')) + result.push_back(c); + } + if (!result.empty() && result.back() == ' ') + result.pop_back(); + return result; +} + +class Pipeline { +public: + Program tagger, verbalizer, postprocessor; + std::string result, batch_result; + bool verbalization_failed = false; + Pipeline(const char *tagger_path, const char *verbalizer_path, + const char *post_path) + : tagger(tagger_path), verbalizer(verbalizer_path), + postprocessor(post_path) { + tagger.push_prefix = true; + postprocessor.project_output = true; + } + + bool rewrite(const char *text, size_t length) { + float cost; + result.clear(); + verbalization_failed = false; + if (!tagger.rewrite(text, length, cost)) + return false; + auto tokens = TagParser(tagger.result).parse(); + for (size_t begin = 0; begin < tokens.size();) { + size_t end = begin, count = 1; + while (end < tokens.size()) { + size_t n = permutation_count({tokens[end]}); + if (n > 729) + throw std::runtime_error("Unsplittable tag permutation group"); + if (count > 729 / n) + break; + count *= n; + ++end; + } + std::vector candidates(1); + for (size_t i = begin; i < end; ++i) { + auto choices = permutations({tokens[i]}); + std::vector expanded; + for (const auto &prefix : candidates) + for (const auto &choice : choices) + expanded.push_back(prefix + choice); + candidates = std::move(expanded); + } + bool found = false; + for (const auto &candidate : candidates) { + if (verbalizer.rewrite(candidate.data(), candidate.size(), cost)) { + if (!result.empty()) + result += ' '; + result += verbalizer.result; + found = true; + break; + } + } + if (!found) { + verbalization_failed = true; + return false; + } + begin = end; + } + result = collapse_spaces(result); + if (!result.empty()) { + if (!postprocessor.rewrite(result.data(), result.size(), cost)) + return false; + result = postprocessor.result; + } + return true; + } +}; +} // namespace +extern "C" { +void *tn_pipeline_load(const char *t, const char *v, const char *p) { + try { + return new Pipeline(t, v, p); + } catch (const std::exception &e) { + error = e.what(); + return nullptr; + } +} +void tn_pipeline_delete(void *p) { delete static_cast(p); } +// Input whitespace/punctuation preprocessing and optional Moses postprocessing +// are outside this experimental core and must be supplied by the caller. +int tn_pipeline_rewrite(void *p, const char *text, size_t length, + const char **out, size_t *size) { + try { + if (!p || !text || !out || !size) + throw std::runtime_error("Null argument"); + auto &pipeline = *static_cast(p); + bool found = pipeline.rewrite(text, length); + *out = pipeline.result.data(); + *size = pipeline.result.size(); + return found ? 0 : 1; + } catch (const std::exception &e) { + error = e.what(); + return -1; + } +} +} + +extern "C" { +int tn_pipeline_normalize(void *p, const char *text, size_t length, + const char **out, size_t *size) { + try { + if (!p || !text || !out || !size) + throw std::runtime_error("Null argument"); + auto &pipeline = *static_cast(p); + auto original = + tn_text::collapse(tn_text::decode(std::string_view(text, length))); + bool found = true; + if (original.empty()) + pipeline.result.clear(); + else { + auto prepared = tn_text::prepare(original); + found = pipeline.rewrite(prepared.data(), prepared.size()); + if (found) + pipeline.result = tn_text::finish(original, pipeline.result); + else if (pipeline.verbalization_failed) { + // NeMo returns its escaped prepared input on failed verbalization. + // It skips punctuation postprocessing on this early-return path. + pipeline.result.clear(); + for (char c : prepared) { + if (c == '[' || c == ']' || c == '\\') + pipeline.result += '\\'; + pipeline.result += c; + } + found = true; + } + } + *out = pipeline.result.data(); + *size = pipeline.result.size(); + return found ? 0 : 1; + } catch (const std::exception &e) { + error = e.what(); + return -1; + } +} +// Consume a bounded batch without Python callbacks between records. +// No partial output is published when any record or frame fails. +int tn_pipeline_normalize_batch(void *p, const char *frames, size_t length, + const char **out, size_t *size) { + if (out) + *out = nullptr; + if (size) + *size = 0; + try { + if (!p || !frames || !out || !size) + throw std::runtime_error("Null argument"); + if (length > 64 * 1024 * 1024) + throw std::runtime_error("Batch exceeds 64 MiB"); + auto &batch = static_cast(p)->batch_result; + batch.clear(); + size_t position = 0, record = 0; + while (position < length) { + if (length - position < 4) + throw std::runtime_error("Truncated batch frame header"); + const auto *h = + reinterpret_cast(frames + position); + uint32_t n = uint32_t(h[0]) | (uint32_t(h[1]) << 8) | + (uint32_t(h[2]) << 16) | (uint32_t(h[3]) << 24); + position += 4; + if (n > 16 * 1024 * 1024 || n > length - position) + throw std::runtime_error("Invalid batch frame length"); + const char *value = nullptr; + size_t value_size = 0; + int status = + tn_pipeline_normalize(p, frames + position, n, &value, &value_size); + if (status) + throw std::runtime_error("Batch record " + std::to_string(record) + + ": " + + (status < 0 ? error : "No accepting path")); + if (value_size > UINT32_MAX || + value_size + 4 > 64 * 1024 * 1024 - batch.size()) + throw std::runtime_error("Batch output exceeds 64 MiB"); + for (size_t i = 0; i < 4; ++i) + batch.push_back(static_cast((value_size >> (8 * i)) & 255)); + batch.append(value, value_size); + position += n; + ++record; + } + *out = batch.data(); + *size = batch.size(); + return 0; + } catch (const std::exception &e) { + if (p) + static_cast(p)->batch_result.clear(); + error = e.what(); + return -1; + } +} +// Independent stage entry points let differential tests isolate text processing +// from weighted-path selection; result memory lasts until the next stage call. +int tn_text_stage(int stage, const char *input, size_t n, const char *output, + size_t m, const char **result, size_t *length) { + try { + if (!input || !output || !result || !length) + throw std::runtime_error("Null argument"); + static thread_local std::string value; + auto original = + tn_text::collapse(tn_text::decode(std::string_view(input, n))); + if (stage == 0) + value = tn_text::prepare(original); + else if (stage == 1) + value = tn_text::encode( + tn_text::detokenize(tn_text::decode(std::string_view(output, m)))); + else if (stage == 2) + value = tn_text::finish(original, std::string_view(output, m)); + else + throw std::runtime_error("Invalid text stage"); + *result = value.data(); + *length = value.size(); + return 0; + } catch (const std::exception &e) { + error = e.what(); + return -1; + } +} +} + +extern "C" int tn_set_mode(void *handle, int mode) { + if (!handle || mode < 0 || mode > 2) { + error = "Invalid evaluator mode"; + return -1; + } + auto &program = *static_cast(handle); + program.push_prefix = mode == 1; + program.project_output = mode == 2; + return 0; +} diff --git a/tools/text_processing_deployment/native_runtime/byte_transducer.py b/tools/text_processing_deployment/native_runtime/byte_transducer.py new file mode 100644 index 000000000..af8623022 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/byte_transducer.py @@ -0,0 +1,210 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Export tropical byte transducers and exercise the experimental native evaluator.""" + +from __future__ import annotations + +import ctypes +import struct +from pathlib import Path +from typing import TYPE_CHECKING, Literal + +if TYPE_CHECKING: + import pynini + + +def export_fst(fst: pynini.Fst, path: Path) -> None: + """Export byte arcs and cached acyclicity used by reference queue selection.""" + import pynini + + if fst.arc_type() != "standard" or fst.weight_type() != "tropical": + raise ValueError("Only standard tropical arcs are supported") + states = list(fst.states()) + if states != list(range(len(states))) or fst.start() < 0: + raise ValueError("Expected dense states and a valid start") + arcs = [] + records = [] + for state in states: + begin = len(arcs) + for arc in sorted(fst.arcs(state), key=lambda a: a.ilabel): + if not 0 <= arc.ilabel <= 255 or not 0 <= arc.olabel <= 255: + raise ValueError("Only epsilon and byte labels are supported") + weight = float(arc.weight) + arcs.append((arc.ilabel, arc.olabel, arc.nextstate, weight)) + records.append((begin, len(arcs), float(fst.final(state)))) + with path.open("wb") as out: + out.write( + struct.pack( + "<8sIIII", + b"TNBYTE02", + len(records), + len(arcs), + fst.start(), + int(bool(fst.properties(pynini.ACYCLIC, False))), + ) + ) + for record in records: + out.write(struct.pack(" None: + """Select direct, prepared-lookahead, or projected-output path semantics.""" + modes = {"transducer": 0, "lookahead": 1, "output": 2} + if mode not in modes: + raise ValueError(f"Unsupported evaluator mode: {mode}") + self.lib = ctypes.CDLL(str(library.absolute())) + self.lib.tn_load.argtypes = [ctypes.c_char_p] + self.lib.tn_load.restype = ctypes.c_void_p + self.lib.tn_delete.argtypes = [ctypes.c_void_p] + self.lib.tn_delete.restype = None + self.lib.tn_error.restype = ctypes.c_char_p + self.lib.tn_rewrite.argtypes = [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_size_t), + ctypes.POINTER(ctypes.c_float), + ] + self.lib.tn_rewrite.restype = ctypes.c_int + self.lib.tn_set_mode.argtypes = [ctypes.c_void_p, ctypes.c_int] + self.lib.tn_set_mode.restype = ctypes.c_int + self.handle = self.lib.tn_load(str(grammar.absolute()).encode()) + if not self.handle: + raise RuntimeError(self.lib.tn_error().decode()) + if self.lib.tn_set_mode(self.handle, modes[mode]): + self.close() + raise RuntimeError(self.lib.tn_error().decode()) + + def close(self) -> None: + """Release native memory.""" + if self.handle: + self.lib.tn_delete(self.handle) + self.handle = None + + def rewrite(self, text: str) -> tuple[str, float] | None: + """Return a minimum-cost output, or None if no accepting path exists.""" + if not self.handle: + raise RuntimeError("Transducer is closed") + encoded = text.encode() + output = ctypes.c_void_p() + size = ctypes.c_size_t() + cost = ctypes.c_float() + status = self.lib.tn_rewrite( + self.handle, + encoded, + len(encoded), + ctypes.byref(output), + ctypes.byref(size), + ctypes.byref(cost), + ) + if status < 0: + raise RuntimeError(self.lib.tn_error().decode()) + if status: + return None + return ctypes.string_at(output.value, size.value).decode(), cost.value + + +class NativePipeline: + """Native classifier, tag parser, reordering, verbalizer and postprocessing FST. + + normalize includes English punctuation handling and Unicode whitespace collapse. + normalize_core exposes the transducer stages for compatibility diagnostics. + """ + + def __init__(self, library: Path, grammar_directory: Path) -> None: + self.lib = ctypes.CDLL(str(library.absolute())) + self.lib.tn_error.restype = ctypes.c_char_p + self.lib.tn_pipeline_load.argtypes = [ctypes.c_char_p] * 3 + self.lib.tn_pipeline_load.restype = ctypes.c_void_p + self.lib.tn_pipeline_delete.argtypes = [ctypes.c_void_p] + self.lib.tn_pipeline_delete.restype = None + self.lib.tn_pipeline_rewrite.argtypes = [ + ctypes.c_void_p, + ctypes.c_char_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_size_t), + ] + self.lib.tn_pipeline_rewrite.restype = ctypes.c_int + self.lib.tn_pipeline_normalize.argtypes = self.lib.tn_pipeline_rewrite.argtypes + self.lib.tn_pipeline_normalize.restype = ctypes.c_int + paths = [ + (grammar_directory / (name + ".bin")).absolute() + for name in ("tagger", "verbalizer", "postprocessor") + ] + self.handle = self.lib.tn_pipeline_load(*(str(path).encode() for path in paths)) + if not self.handle: + raise RuntimeError(self.lib.tn_error().decode()) + + def close(self) -> None: + """Release native grammar and workspace storage.""" + if self.handle: + self.lib.tn_pipeline_delete(self.handle) + self.handle = None + + def normalize_core(self, text: str) -> str | None: + """Normalize preprocessed UTF-8 input with no Python work between stages.""" + return self._normalize(text, complete=False) + + def normalize(self, text: str) -> str | None: + """Run all English normalization stages, including punctuation, in native code.""" + return self._normalize(text, complete=True) + + def normalize_batch(self, texts: list[str]) -> list[str]: + """Run a bounded batch in native code and fail without returning partial results.""" + if not self.handle: + raise RuntimeError("Pipeline is closed") + encoded = [text.encode() for text in texts] + frames = b"".join(struct.pack(" len(data) - position: + raise RuntimeError("Truncated native batch record") + result.append(data[position : position + length].decode()) + position += length + if len(result) != len(texts): + raise RuntimeError("Native batch record count changed") + return result + + def _normalize(self, text: str, *, complete: bool) -> str | None: + if not self.handle: + raise RuntimeError("Pipeline is closed") + encoded = text.encode() + output = ctypes.c_void_p() + size = ctypes.c_size_t() + function = ( + self.lib.tn_pipeline_normalize if complete else self.lib.tn_pipeline_rewrite + ) + status = function( + self.handle, encoded, len(encoded), ctypes.byref(output), ctypes.byref(size) + ) + if status < 0: + raise RuntimeError(self.lib.tn_error().decode()) + if status: + return None + return ctypes.string_at(output.value, size.value).decode() diff --git a/tools/text_processing_deployment/native_runtime/export_grammars.py b/tools/text_processing_deployment/native_runtime/export_grammars.py new file mode 100644 index 000000000..9fe413314 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/export_grammars.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Export an existing deterministic English NeMo normalizer for native evaluation.""" + +from argparse import ArgumentParser +from pathlib import Path + +from byte_transducer import export_fst +from nemo_text_processing.text_normalization.normalize import Normalizer + + +def main() -> None: + parser = ArgumentParser(description=__doc__) + parser.add_argument("--cache-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + normalizer = Normalizer( + input_case="cased", + lang="en", + cache_dir=str(args.cache_dir), + overwrite_cache=False, + ) + args.output_dir.mkdir(parents=True, exist_ok=True) + for name, grammar in ( + ("tagger", normalizer.tagger.fst), + ("verbalizer", normalizer.verbalizer.fst), + ("postprocessor", normalizer.post_processor.fst), + ): + export_fst(grammar, args.output_dir / (name + ".bin")) + + +if __name__ == "__main__": + main() diff --git a/tools/text_processing_deployment/native_runtime/export_text_tables.py b/tools/text_processing_deployment/native_runtime/export_text_tables.py new file mode 100644 index 000000000..7f8b948c5 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/export_text_tables.py @@ -0,0 +1,52 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Export the installed Python/sacremoses character properties at build time.""" + +import unicodedata +from argparse import ArgumentParser +from importlib.metadata import version +from pathlib import Path + +from sacremoses import MosesDetokenizer +from sacremoses.util import CJKChars + + +def ranges(values: list[int]) -> list[tuple[int, int]]: + """Coalesce exact codepoint membership into sorted closed intervals.""" + result: list[tuple[int, int]] = [] + for value in sorted(set(values)): + if result and result[-1][1] + 1 == value: + result[-1] = (result[-1][0], value) + else: + result.append((value, value)) + return result + + +def export(path: Path) -> None: + """Write immutable C++ tables; runtime processing needs no Python.""" + properties = { + "whitespace": ranges([c for c in range(0x110000) if chr(c).isspace()]), + "alpha": ranges(list(map(ord, MosesDetokenizer.IsAlpha))), + "currency_symbols": ranges(list(map(ord, MosesDetokenizer.IsSc))), + "cjk": CJKChars.ranges, + } + lines = [ + "// Generated by export_text_tables.py; do not edit.", + f"// Python Unicode {unicodedata.unidata_version}; sacremoses {version('sacremoses')}.", + "// Sacremoses-derived tables are covered by LICENSE.sacremoses.", + "#pragma once", + "namespace tn_text {", + "struct Range { char32_t first, last; };", + ] + for name, items in properties.items(): + lines.append(f"inline constexpr Range {name}[] = {{") + lines.extend(f" {{0x{a:x},0x{b:x}}}," for a, b in items) + lines.append("};") + lines.extend(["}", ""]) + path.write_text("\n".join(lines)) + + +if __name__ == "__main__": + parser = ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + export(parser.parse_args().output) diff --git a/tools/text_processing_deployment/native_runtime/normalize_stream.cc b/tools/text_processing_deployment/native_runtime/normalize_stream.cc new file mode 100644 index 000000000..686d8ed7d --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/normalize_stream.cc @@ -0,0 +1,82 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +// Binary framing keeps embedded newlines intact and amortizes process startup. +// Each record is a little-endian uint32 byte length followed by UTF-8 bytes. +#include +#include +#include +#include +#include +#include +#include +extern "C" { +void *tn_pipeline_load(const char *, const char *, const char *); +void tn_pipeline_delete(void *); +const char *tn_error(); +int tn_pipeline_normalize(void *, const char *, size_t, const char **, + size_t *); +} +int main(int argc, char **argv) { + try { + if (argc != 4) + throw std::runtime_error("Usage: normalize_stream GRAMMAR_DIRECTORY " + "INPUT.frames OUTPUT.frames"); + std::string root = argv[1]; + std::unique_ptr pipeline( + tn_pipeline_load((root + "/tagger.bin").c_str(), + (root + "/verbalizer.bin").c_str(), + (root + "/postprocessor.bin").c_str()), + tn_pipeline_delete); + if (!pipeline) + throw std::runtime_error(tn_error()); + std::ifstream in(argv[2], std::ios::binary); + std::ofstream out(argv[3], std::ios::binary); + if (!in || !out) + throw std::runtime_error("Cannot open input/output"); + auto start = std::chrono::steady_clock::now(); + size_t count = 0; + std::string input; + for (;;) { + unsigned char h[4]; + in.read(reinterpret_cast(h), 4); + if (in.gcount() == 0 && in.eof()) + break; + if (in.gcount() != 4) + throw std::runtime_error("Truncated input frame header"); + uint32_t n = uint32_t(h[0]) | (uint32_t(h[1]) << 8) | + (uint32_t(h[2]) << 16) | (uint32_t(h[3]) << 24); + if (n > 16 * 1024 * 1024) + throw std::runtime_error("Input frame exceeds 16 MiB"); + input.resize(n); + if (!in.read(input.data(), n)) + throw std::runtime_error("Truncated input frame"); + const char *result = nullptr; + size_t length = 0; + int status = tn_pipeline_normalize(pipeline.get(), input.data(), + input.size(), &result, &length); + if (status) + throw std::runtime_error(status < 0 ? tn_error() : "No accepting path"); + if (length > UINT32_MAX) + throw std::runtime_error("Output frame too long"); + for (size_t i = 0; i < 4; ++i) + h[i] = (length >> (8 * i)) & 255; + out.write(reinterpret_cast(h), 4); + out.write(result, length); + if (!out) + throw std::runtime_error("Output write failed"); + ++count; + } + out.flush(); + if (!out) + throw std::runtime_error("Output flush failed"); + std::cerr << "{\"records\":" << count << ",\"processing_seconds\":" + << std::chrono::duration( + std::chrono::steady_clock::now() - start) + .count() + << "}\n"; + return 0; + } catch (const std::exception &e) { + std::cerr << e.what() << '\n'; + return 1; + } +} diff --git a/tools/text_processing_deployment/native_runtime/test_byte_transducer.py b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py new file mode 100644 index 000000000..7d9c74734 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py @@ -0,0 +1,420 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compare the native evaluator against OpenFst on bounded weighted graphs.""" + +import random +import subprocess +import tempfile +import unittest +from pathlib import Path + +import pynini +from .byte_transducer import ByteTransducer, NativePipeline, export_fst + + +class ByteTransducerTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.directory = tempfile.TemporaryDirectory() + cls.root = Path(cls.directory.name) + cls.library = cls.root / "libtn_byte_transducer.so" + subprocess.run( + [ + "g++", + "-std=c++17", + "-O3", + "-Wall", + "-Wextra", + "-shared", + "-fPIC", + str(Path(__file__).with_suffix(".cc").with_name("byte_transducer.cc")), + "-o", + str(cls.library), + ], + check=True, + ) + + @classmethod + def tearDownClass(cls) -> None: + cls.directory.cleanup() + + def check_graph(self, graph: pynini.Fst, inputs: list[str]) -> None: + path = self.root / "grammar.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path) + try: + for text in inputs: + with self.subTest(text=text): + actual = runtime.rewrite(text) + lattice = pynini.compose(pynini.accep(pynini.escape(text)), graph) + expected_cost = ( + float( + pynini.shortestdistance(lattice, reverse=True)[ + lattice.start() + ] + ) + if lattice.start() >= 0 + else float("inf") + ) + self.assertEqual(actual is None, expected_cost == float("inf")) + if actual is not None: + output, cost = actual + self.assertAlmostEqual(cost, expected_cost, places=5) + constrained = pynini.compose( + lattice, pynini.accep(pynini.escape(output)) + ) + self.assertGreaterEqual(constrained.start(), 0) + output_cost = float( + pynini.shortestdistance(constrained, reverse=True)[ + constrained.start() + ] + ) + self.assertAlmostEqual(output_cost, expected_cost, places=5) + finally: + runtime.close() + + def test_random_acyclic_weighted_graphs(self) -> None: + rng = random.Random(81) + for _ in range(50): + graph = pynini.Fst() + for _ in range(12): + graph.add_state() + graph.set_start(0) + graph.set_final(11, pynini.Weight("tropical", rng.randrange(-3, 4) / 8)) + for source in range(11): + for target in range(source + 1, 12): + if rng.random() < 0.22 or target == source + 1: + graph.add_arc( + source, + pynini.Arc( + rng.choice([0, 97, 98]), + rng.choice([0, 120, 121]), + pynini.Weight("tropical", rng.randrange(-8, 9) / 8), + target, + ), + ) + self.check_graph(graph, ["", "a", "b", "ab", "ba", "aaa", "abab"]) + + def test_unicode_and_literal_metacharacters(self) -> None: + graph = pynini.union( + pynini.cross(pynini.escape("café [x]"), pynini.escape("café [output]")), + pynini.cross("12", "twelve"), + pynini.cross("", "empty"), + ) + self.check_graph(graph, ["café [x]", "12", "", "no match", "12"]) + + def test_consuming_cycles(self) -> None: + graph = pynini.closure(pynini.cross("a", "xyz")) + self.check_graph(graph, ["", "a", "aaaa", "b", "a"]) + + def test_epsilon_cycle_with_nonnegative_cost(self) -> None: + graph = pynini.Fst() + graph.add_state() + graph.add_state() + graph.set_start(0) + graph.set_final(1) + graph.add_arc(0, pynini.Arc(0, 120, pynini.Weight("tropical", 1), 0)) + graph.add_arc(0, pynini.Arc(97, 121, pynini.Weight("tropical", 0), 1)) + self.check_graph(graph, ["a", "", "aa", "a"]) + + def test_invalid_and_closed_handles(self) -> None: + path = self.root / "invalid.bin" + path.write_bytes(b"not a transducer") + with self.assertRaises(RuntimeError): + ByteTransducer(self.library, path) + export_fst(pynini.accep("x"), path) + runtime = ByteTransducer(self.library, path) + runtime.close() + runtime.close() + with self.assertRaises(RuntimeError): + runtime.rewrite("x") + + def test_native_pipeline_field_reordering(self) -> None: + tags = 'tokens { money { currency: "dollars" integer: "twelve" } }' + reordered = ' tokens { money { integer: "twelve" currency: "dollars" } } ' + export_fst(pynini.cross("input", tags), self.root / "tagger.bin") + export_fst( + pynini.cross(reordered, "twelve dollars"), self.root / "verbalizer.bin" + ) + export_fst( + pynini.cross("twelve dollars", "TWELVE DOLLARS"), + self.root / "postprocessor.bin", + ) + native = NativePipeline(self.library, self.root) + try: + self.assertEqual(native.normalize_core("input"), "TWELVE DOLLARS") + self.assertIsNone(native.normalize_core("no path")) + self.assertEqual(native.normalize_core("input"), "TWELVE DOLLARS") + finally: + native.close() + + def test_native_pipeline_preserves_explicit_field_order(self) -> None: + tags = 'tokens { money { currency: "dollars" integer: "twelve" preserve_order: true } }' + forbidden = ' tokens { money { integer: "twelve" currency: "dollars" preserve_order: true } } ' + export_fst(pynini.cross("input", tags), self.root / "tagger.bin") + export_fst( + pynini.cross(forbidden, "twelve dollars"), self.root / "verbalizer.bin" + ) + export_fst(pynini.accep("twelve dollars"), self.root / "postprocessor.bin") + native = NativePipeline(self.library, self.root) + try: + self.assertIsNone(native.normalize_core("input")) + finally: + native.close() + + def test_nonbyte_labels_are_rejected(self) -> None: + graph = pynini.Fst() + graph.add_state() + graph.add_state() + graph.set_start(0) + graph.set_final(1) + graph.add_arc(0, pynini.Arc(1000, 97, pynini.Weight("tropical", 0), 1)) + with self.assertRaises(ValueError): + export_fst(graph, self.root / "unsupported.bin") + + def test_float_tie_after_shared_merge(self) -> None: + graph = pynini.Fst() + for _ in range(4): + graph.add_state() + graph.set_start(0) + graph.set_final(3) + for src, dst, ilabel, olabel, weight in [ + (0, 1, 97, 72, 10.0), + (0, 2, 97, 76, 9.0), + (2, 1, 0, 0, 0.0), + (1, 3, 0, 0, 1e8), + ]: + graph.add_arc( + src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst) + ) + path = self.root / "merge.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path) + try: + expected = pynini.shortestpath(pynini.accep("a") @ graph).string() + self.assertEqual(expected, "L") + self.assertEqual(runtime.rewrite("a")[0], expected) + finally: + runtime.close() + + def test_prepared_lookahead_arc_order(self) -> None: + graph = pynini.Fst() + for _ in range(5): + graph.add_state() + graph.set_start(0) + graph.set_final(4) + for src, dst, ilabel, olabel, weight in [ + (0, 1, 0, 0, 1.0), + (1, 2, 0, 120, 0.0), + (1, 3, 97, 121, 0.0), + (2, 4, 97, 122, 0.0), + (3, 4, 0, 113, 0.0), + ]: + graph.add_arc( + src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst) + ) + path = self.root / "lookahead.bin" + export_fst(graph, path) + for mode, expected in [("transducer", "yq"), ("lookahead", "xz")]: + runtime = ByteTransducer(self.library, path, mode=mode) + try: + self.assertEqual(runtime.rewrite("a")[0], expected) + finally: + runtime.close() + + def test_projected_output_against_top_rewrite(self) -> None: + from pynini.lib import rewrite + + rng = random.Random(173) + for _ in range(30): + graph = pynini.Fst() + for _ in range(10): + graph.add_state() + graph.set_start(0) + graph.set_final(9) + for src in range(9): + for dst in range(src + 1, 10): + if dst == src + 1 or rng.random() < 0.2: + graph.add_arc( + src, + pynini.Arc( + rng.choice([0, 97, 98]), + rng.choice([0, 120, 121]), + pynini.Weight("tropical", rng.randrange(-3, 4) / 8), + dst, + ), + ) + path = self.root / "projected.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path, mode="output") + try: + for text in ["", "a", "b", "ab", "ba", "aaa", "abab"]: + try: + expected = rewrite.top_rewrite(text, graph) + except rewrite.Error: + expected = None + actual = runtime.rewrite(text) + with self.subTest(text=text): + self.assertEqual( + None if actual is None else actual[0], expected + ) + finally: + runtime.close() + + def test_lookahead_phase_merge_before_float_rounding(self) -> None: + graph = pynini.Fst() + for _ in range(5): + graph.add_state() + graph.set_start(0) + for state, weight in [(0, 0), (1, 1), (4, 0)]: + graph.set_final(state, pynini.Weight("tropical", weight)) + for src, dst, ilabel, olabel, weight in [ + (0, 4, 98, 120, -1e-8), + (0, 2, 98, 122, 0), + (2, 3, 98, 122, 0), + (3, 1, 98, 121, -10), + (4, 2, 0, 120, 0), + ]: + graph.add_arc( + src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst) + ) + path = self.root / "phase-merge.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path, mode="lookahead") + try: + # The prepared OpenFst reference retains the cheaper prefix at the merge. + # Compressing the next path first rounds both totals to the same float. + self.assertEqual(runtime.rewrite("bbb"), ("xxzy", -9.0)) + finally: + runtime.close() + + def test_compatibility_modes_reject_epsilon_cycles(self) -> None: + graph = pynini.closure(pynini.cross("", "x")) + pynini.cross("a", "y") + path = self.root / "cycle.bin" + export_fst(graph, path) + for mode in ["lookahead", "output"]: + runtime = ByteTransducer(self.library, path, mode=mode) + try: + with self.assertRaisesRegex(RuntimeError, "Input-epsilon cycles"): + runtime.rewrite("a") + finally: + runtime.close() + + def test_random_exact_outputs(self) -> None: + from pynini.lib import rewrite + + rng = random.Random(20261017) + for case in range(1200): + graph = pynini.Fst() + n = rng.randrange(4, 15) + for _ in range(n): + graph.add_state() + ids = list(range(n)) + if case % 2: + rng.shuffle(ids) + graph.set_start(ids[0]) + for pos, state in enumerate(ids): + if pos == n - 1 or rng.random() < 0.15: + graph.set_final( + state, + pynini.Weight("tropical", rng.choice([0, 0, 0.125, 1, 2])), + ) + for later in range(pos + 1, n): + if later == pos + 1 or rng.random() < 0.25: + graph.add_arc( + state, + pynini.Arc( + rng.choice([0, 0, 97, 98]), + rng.choice([0, 120, 121, 122]), + pynini.Weight( + "tropical", + 0 + if case >= 600 + else rng.choice( + [0, 0, 0, -1e-8, 0.1, 0.2, 1e8, -10] + ), + ), + ids[later], + ), + ) + if case >= 600: + if case % 3 == 0: + target = graph.add_state() + graph.set_final(target) + graph.add_arc( + graph.start(), + pynini.Arc(99, 120, pynini.Weight("tropical", 1), target), + ) + if case % 2: + graph.properties(pynini.ACYCLIC, True) + graph.arcsort("ilabel") + path = self.root / "random-exact.bin" + export_fst(graph, path) + for mode in ["transducer", "output"]: + runtime = ByteTransducer(self.library, path, mode=mode) + try: + for text in ["", "a", "b", "ab", "ba", "aa", "bbb", "abab"]: + if mode == "output": + try: + expected = rewrite.top_rewrite(text, graph) + except rewrite.Error: + expected = None + else: + lattice = pynini.accep(text) @ graph + expected = ( + pynini.shortestpath(lattice).string() + if lattice.start() >= 0 + else None + ) + actual = runtime.rewrite(text) + with self.subTest(case=case, mode=mode, text=text): + self.assertEqual( + None if actual is None else actual[0], expected + ) + finally: + runtime.close() + + def test_unweighted_reference_queue_order(self) -> None: + # OpenFst AutoQueue chooses LIFO when acyclicity is not already known. + graph = pynini.Fst() + for _ in range(7): + graph.add_state() + graph.set_start(2) + graph.set_final(4) + for src, dst, ilabel, olabel in [ + (0, 1, 0, 122), + (0, 4, 98, 122), + (1, 4, 98, 120), + (2, 0, 0, 120), + (2, 1, 0, 121), + ]: + graph.add_arc( + src, pynini.Arc(ilabel, olabel, pynini.Weight.one("tropical"), dst) + ) + graph.arcsort("ilabel") + expected = pynini.shortestpath(pynini.accep("b") @ graph).string() + self.assertEqual(expected, "yx") + path = self.root / "unweighted.bin" + for known_acyclic in [False, True]: + for unused_weighted_branch in [False, True]: + variant = graph.copy() + if unused_weighted_branch: + target = variant.add_state() + variant.set_final(target) + variant.add_arc( + variant.start(), + pynini.Arc(99, 120, pynini.Weight("tropical", 1), target), + ) + if known_acyclic: + variant.properties(pynini.ACYCLIC, True) + expected = pynini.shortestpath(pynini.accep("b") @ variant).string() + export_fst(variant, path) + runtime = ByteTransducer(self.library, path) + try: + self.assertEqual(runtime.rewrite("b")[0], expected) + finally: + runtime.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/text_processing_deployment/native_runtime/test_text_processing.py b/tools/text_processing_deployment/native_runtime/test_text_processing.py new file mode 100644 index 000000000..c0b6cd318 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/test_text_processing.py @@ -0,0 +1,295 @@ +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Differential tests for native English text stages against their Python references.""" + +import ctypes +import logging +import random +import struct +import subprocess +import tempfile +import unittest +from pathlib import Path + +import pynini +from .byte_transducer import NativePipeline, export_fst +from nemo_text_processing.text_normalization.data_loader_utils import ( + post_process_punct, + pre_process, +) +from sacremoses import MosesDetokenizer + + +class TextProcessingTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + cls.previous_logging_disable = logging.root.manager.disable + logging.disable(logging.CRITICAL) + cls.directory = tempfile.TemporaryDirectory() + cls.root = Path(cls.directory.name) + source = Path(__file__).parent + cls.library = cls.root / "native.so" + cls.scalar = cls.root / "scalar.so" + for library, flags in ((cls.library, []), (cls.scalar, ["-DTN_DISABLE_SIMD"])): + subprocess.run( + [ + "g++", + "-std=c++17", + "-O3", + "-Wall", + "-Wextra", + "-shared", + "-fPIC", + *flags, + str(source / "byte_transducer.cc"), + "-o", + str(library), + ], + check=True, + ) + cls.libs = [ctypes.CDLL(str(p)) for p in (cls.library, cls.scalar)] + for lib in cls.libs: + lib.tn_text_stage.argtypes = [ + ctypes.c_int, + ctypes.c_char_p, + ctypes.c_size_t, + ctypes.c_char_p, + ctypes.c_size_t, + ctypes.POINTER(ctypes.c_void_p), + ctypes.POINTER(ctypes.c_size_t), + ] + lib.tn_text_stage.restype = ctypes.c_int + cls.detokenizer = MosesDetokenizer(lang="en") + cls.executable = cls.root / "normalize_stream" + subprocess.run( + [ + "g++", + "-std=c++17", + "-O3", + str(source / "byte_transducer.cc"), + str(source / "normalize_stream.cc"), + "-o", + str(cls.executable), + ], + check=True, + ) + + @classmethod + def tearDownClass(cls) -> None: + cls.directory.cleanup() + logging.disable(cls.previous_logging_disable) + + def stage(self, stage: int, written: str, output: str) -> str: + a, b = written.encode(), output.encode() + results = [] + for lib in self.libs: + result, length = ctypes.c_void_p(), ctypes.c_size_t() + status = lib.tn_text_stage( + stage, a, len(a), b, len(b), ctypes.byref(result), ctypes.byref(length) + ) + self.assertEqual(status, 0) + results.append(ctypes.string_at(result.value, length.value).decode()) + self.assertEqual(results[0], results[1], "SIMD/scalar mismatch") + return results[0] + + def check_pair(self, written: str, output: str) -> None: + written = " ".join(written.split()) + self.assertEqual(self.stage(0, written, ""), pre_process(written).strip()) + detokenized = self.detokenizer.detokenize([output], unescape=False) + self.assertEqual(self.stage(1, "", output), detokenized) + expected = " ".join( + post_process_punct(input=written, normalized_text=detokenized).split() + ) + self.assertEqual(self.stage(2, written, output), expected) + + def test_public_punctuation_and_unicode_cases(self) -> None: + pairs = [ + ("[12] test' example", "[ twelve ] test ' example"), + ("The Jones' house.", "The Jones ' house ."), + (chr(96) * 2 + "A test''", "\" A test ' '"), + ("a:b:c", "a : b c"), + ("a:: b", "a : : b"), + ("a [x] b", "a [ x ] b"), + ("a\u00a0b\u2003c", "a\u00a0b\u2003c"), + ("$5.00 and €3", "$ five . zero zero and € three"), + ("你好 世界 한 국", "你好 世界 한 국"), + ("a @-@ b @-@ c", "a @-@ b @-@ c"), + ("' é ' 中文", "' é ' 中文"), + ("'\U00010400", "'\U00010400"), + ("", ""), + ] + for a, b in pairs: + with self.subTest(written=a, output=b): + self.check_pair(a, b) + + def test_seeded_text_stage_equivalence(self) -> None: + rng = random.Random(914) + units = [ + "word", + "James", + "café", + "你好", + "국", + "'s", + "'é", + "a", + "", + " ", + "\t", + "\u00a0", + "\u2003", + "[", + "]", + "(", + ")", + "$", + "€", + "¿", + "¡", + ":", + ";", + ".", + ",", + "!", + "?", + "\\", + "%", + "'", + '"', + chr(96) * 2, + "''", + "„", + "“", + "”", + "a:b", + "@-@", + "\U00010400", + "\u1100", + "\uffdc", + "\U0002ffff", + ] + for _ in range(2000): + a = " ".join(rng.choices(units, k=rng.randrange(30))) + b = " ".join(rng.choices(units, k=rng.randrange(30))) + with self.subTest(written=a, output=b): + self.check_pair(a, b) + + def test_invalid_utf8_and_nul(self) -> None: + for value in ( + b"\xc0\x80", + b"\xed\xa0\x80", + b"\xf4\x90\x80\x80", + b"\xff", + b"\xe2\x82", + b"\0", + b"a" * 16 + b"\0", + ): + for lib in self.libs: + result, length = ctypes.c_void_p(), ctypes.c_size_t() + status = lib.tn_text_stage( + 0, + value, + len(value), + b"", + 0, + ctypes.byref(result), + ctypes.byref(length), + ) + self.assertEqual(status, -1) + + def test_stream_and_complete_api(self) -> None: + export_fst( + pynini.cross("x", 'tokens { name: "word" }'), self.root / "tagger.bin" + ) + export_fst( + pynini.cross(' tokens { name: "word" } ', "word"), + self.root / "verbalizer.bin", + ) + export_fst(pynini.accep("word"), self.root / "postprocessor.bin") + values = ["x", " \nx\t", "", "\u2003x\u00a0"] + native = NativePipeline(self.library, self.root) + try: + self.assertEqual( + [native.normalize(s) for s in values], ["word", "word", "", "word"] + ) + self.assertEqual( + native.normalize_batch(values), ["word", "word", "", "word"] + ) + self.assertEqual(native.normalize_batch([]), []) + with self.assertRaisesRegex(RuntimeError, "Batch record 1"): + native.normalize_batch(["x", "\0", "x"]) + self.assertEqual(native.normalize_batch(["x"]), ["word"]) + for malformed in [b"\x02\x00", struct.pack(" None: + from types import SimpleNamespace + + from nemo_text_processing.text_normalization.normalize import Normalizer + from nemo_text_processing.text_normalization.token_parser import TokenParser + + normalizer = Normalizer.__new__(Normalizer) + normalizer.lang = "en" + normalizer.parser = TokenParser() + normalizer.max_number_of_permutations_per_split = 729 + normalizer.moses_detokenizer = self.detokenizer + normalizer.verbalizer = SimpleNamespace(fst=pynini.cross("unreachable", "word")) + normalizer.post_processor = SimpleNamespace(fst=pynini.accep("word")) + for tags in ['tokens { name: "word" }', 'tokens { name: "" }']: + normalizer.tagger = SimpleNamespace( + fst=pynini.cross(pynini.escape("[ x ]"), tags) + ) + for name, component in [ + ("tagger", normalizer.tagger), + ("verbalizer", normalizer.verbalizer), + ("postprocessor", normalizer.post_processor), + ]: + export_fst(component.fst, self.root / (name + ".bin")) + native = NativePipeline(self.library, self.root) + try: + expected = normalizer.normalize( + "[x]", punct_pre_process=True, punct_post_process=True + ) + self.assertEqual(native.normalize("[x]"), expected) + finally: + native.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/text_processing_deployment/native_runtime/text_processing.h b/tools/text_processing_deployment/native_runtime/text_processing.h new file mode 100644 index 000000000..d4deb4703 --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/text_processing.h @@ -0,0 +1,319 @@ +// Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 AND MIT +// English detokenization follows sacremoses; see LICENSE.sacremoses. +// Punctuation restoration follows NeMo's character-slot semantics, including deletions. +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include +#if defined(__aarch64__) && !defined(TN_DISABLE_SIMD) +#include +#elif defined(__SSE2__) && !defined(TN_DISABLE_SIMD) +#include +#endif +#include "text_tables.h" + +namespace tn_text { +template bool contains(char32_t c, const Range (&ranges)[N]) { + auto p = + std::lower_bound(std::begin(ranges), std::end(ranges), c, + [](const Range &r, char32_t v) { return r.last < v; }); + return p != std::end(ranges) && p->first <= c; +} +bool space(char32_t c) { + if (c < 128) + return c == ' ' || (c >= 9 && c <= 13) || (c >= 28 && c <= 31); + return contains(c, whitespace); +} +std::u32string decode(std::string_view s) { + std::u32string out; + out.reserve(s.size()); + size_t i = 0; + while (i < s.size()) { + // Widen ASCII blocks without a branch or UTF-8 decoder call per character. +#if defined(__aarch64__) && !defined(TN_DISABLE_SIMD) + if (i + 16 <= s.size()) { + auto v = vld1q_u8(reinterpret_cast(s.data() + i)); + if (vmaxvq_u8(v) < 128) { + size_t n = out.size(); + out.resize(n + 16); + auto lo = vmovl_u8(vget_low_u8(v)), hi = vmovl_u8(vget_high_u8(v)); + auto *p = reinterpret_cast(out.data() + n); + vst1q_u32(p, vmovl_u16(vget_low_u16(lo))); + vst1q_u32(p + 4, vmovl_u16(vget_high_u16(lo))); + vst1q_u32(p + 8, vmovl_u16(vget_low_u16(hi))); + vst1q_u32(p + 12, vmovl_u16(vget_high_u16(hi))); + i += 16; + continue; + } + } +#elif defined(__SSE2__) && !defined(TN_DISABLE_SIMD) + if (i + 16 <= s.size()) { + auto v = _mm_loadu_si128(reinterpret_cast(s.data() + i)); + if (!_mm_movemask_epi8(v)) { + size_t n = out.size(); + out.resize(n + 16); + auto z = _mm_setzero_si128(), lo = _mm_unpacklo_epi8(v, z), + hi = _mm_unpackhi_epi8(v, z); + auto *p = reinterpret_cast<__m128i *>(out.data() + n); + _mm_storeu_si128(p, _mm_unpacklo_epi16(lo, z)); + _mm_storeu_si128(p + 1, _mm_unpackhi_epi16(lo, z)); + _mm_storeu_si128(p + 2, _mm_unpacklo_epi16(hi, z)); + _mm_storeu_si128(p + 3, _mm_unpackhi_epi16(hi, z)); + i += 16; + continue; + } + } +#endif + uint32_t c = static_cast(s[i++]), minimum = 0; + size_t extra = 0; + if (c < 128) { + out.push_back(c); + continue; + } + if (c >= 0xc2 && c <= 0xdf) { + c &= 31; + extra = 1; + minimum = 0x80; + } else if (c >= 0xe0 && c <= 0xef) { + c &= 15; + extra = 2; + minimum = 0x800; + } else if (c >= 0xf0 && c <= 0xf4) { + c &= 7; + extra = 3; + minimum = 0x10000; + } else + throw std::runtime_error("Invalid UTF-8 lead byte"); + if (i + extra > s.size()) + throw std::runtime_error("Truncated UTF-8"); + for (size_t j = 0; j < extra; ++j) { + uint8_t b = static_cast(s[i++]); + if ((b & 0xc0) != 0x80) + throw std::runtime_error("Invalid UTF-8 continuation"); + c = (c << 6) | (b & 63); + } + if (c < minimum || c > 0x10ffff || (c >= 0xd800 && c <= 0xdfff)) + throw std::runtime_error("Invalid UTF-8 code point"); + out.push_back(c); + } + if (out.find(U'\0') != std::u32string::npos) + throw std::runtime_error("Embedded NUL is unsupported"); + return out; +} +std::string encode(const std::u32string &s) { + std::string out; + out.reserve(s.size()); + for (uint32_t c : s) { + if (c < 0x80) + out.push_back(c); + else if (c < 0x800) { + out.push_back(0xc0 | (c >> 6)); + out.push_back(0x80 | (c & 63)); + } else if (c < 0x10000) { + out.push_back(0xe0 | (c >> 12)); + out.push_back(0x80 | ((c >> 6) & 63)); + out.push_back(0x80 | (c & 63)); + } else { + out.push_back(0xf0 | (c >> 18)); + out.push_back(0x80 | ((c >> 12) & 63)); + out.push_back(0x80 | ((c >> 6) & 63)); + out.push_back(0x80 | (c & 63)); + } + } + return out; +} +std::u32string collapse(const std::u32string &s) { + std::u32string out; + out.reserve(s.size()); + bool pending = false; + for (char32_t c : s) { + if (space(c)) { + pending = !out.empty(); + continue; + } + if (pending) + out += U' '; + pending = false; + out += c; + } + return out; +} +std::u32string replace_all(const std::u32string &s, std::u32string_view from, + std::u32string_view to) { + std::u32string out; + out.reserve(s.size()); + size_t p = 0; + while (p < s.size()) { + size_t k = s.find(from, p); + if (k == std::u32string::npos) { + out.append(s, p, s.size() - p); + break; + } + out.append(s, p, k - p); + out += to; + p = k + from.size(); + } + return out; +} +std::string prepare(const std::u32string &input) { + std::u32string out; + out.reserve(input.size() + 8); + for (char32_t c : input) { + if (c == U'[' || c == U']') { + out += U' '; + out += c; + out += U' '; + } else + out += c; + } + return encode(collapse(out)); +} +bool all_of(std::u32string_view s, std::u32string_view set) { + return std::all_of(s.begin(), s.end(), + [&](char32_t c) { return set.find(c) != set.npos; }); +} +std::u32string detokenize(const std::u32string &input) { + auto text = replace_all(U" " + input + U" ", U" @-@ ", U"-"); + std::vector tokens; + for (size_t i = 0; i < text.size();) { + if (space(text[i])) { + ++i; + continue; + } + size_t begin = i; + while (i < text.size() && !space(text[i])) + ++i; + tokens.emplace_back(text.data() + begin, i - begin); + } + std::u32string out; + out.reserve(text.size()); + bool prepend = true; + std::unordered_map quotes; + for (size_t i = 0; i < tokens.size(); ++i) { + auto token = tokens[i]; + bool currency = std::all_of(token.begin(), token.end(), [](char32_t c) { + return contains(c, currency_symbols) || + std::u32string_view(U"([{¿¡").find(c) != std::u32string_view::npos; + }); + if (contains(token.front(), cjk)) { + if ((!i || !contains(tokens[i - 1].back(), cjk)) && prepend) + out += U' '; + out += token; + prepend = true; + } else if (currency) { + if (prepend) + out += U' '; + out += token; + prepend = false; + } else if (all_of(token, U",.?!:;\\%}])") || + (i && token.size() > 1 && token[0] == U'\'' && + contains(token[1], alpha))) { + out += token; + prepend = true; + } else if (all_of(token, U"'\"„“\x60")) { + std::u32string key = + all_of(token, U"„“”") ? U"\"" : std::u32string(token); + auto &count = quotes[key]; + if (count % 2 == 0) { + if (token == U"'" && i && tokens[i - 1].back() == U's') { + out += token; + prepend = true; + } else { + if (prepend) + out += U' '; + out += token; + prepend = false; + ++count; + } + } else { + out += token; + prepend = true; + ++count; + } + } else { + if (prepend) + out += U' '; + out += token; + prepend = true; + } + } + return collapse(out); +} +struct Cell { + char32_t value; + bool appended_space = false; + bool equals(char32_t c) const { return value == c && !appended_space; } + void append_space() { + if (!value) { + value = U' '; + appended_space = false; + } else + appended_space = true; + } +}; +std::u32string restore(std::u32string input, const std::u32string &normalized) { + if (input.find(U"\x60\x60") != input.npos && + normalized.find(U"\x60\x60") == normalized.npos) + input = replace_all(input, U"\x60\x60", U"\""); + std::vector cells; + cells.reserve(normalized.size()); + for (auto c : normalized) + cells.push_back({c, false}); + // Preserve Python's stable character slots: removal does not shift later indices. + for (char32_t punct : + std::u32string_view(U"!\"#$%&'()*+,-./:;<=>?@[\\]^_\x60{|}~")) { + size_t count_in = std::count(input.begin(), input.end(), punct); + if (!count_in) + continue; + size_t count_out = + std::count_if(cells.begin(), cells.end(), + [&](const Cell &c) { return c.equals(punct); }); + bool equal = count_in == count_out; + size_t j = 0; + for (size_t i = 0; i < input.size(); ++i) { + if (input[i] != punct) + continue; + while (j < cells.size() && !cells[j].equals(punct)) + ++j; + if (j == cells.size()) + break; + bool valid = (i && j && cells[j - 1].equals(input[i - 1])) || + (i + 1 < input.size() && j + 1 < cells.size() && + cells[j + 1].equals(input[i + 1])); + if (!equal && !valid) + continue; + if (i && j) { + if (cells[j - 1].equals(U' ') && input[i - 1] != U' ') + cells[j - 1] = {0, false}; + else if (!cells[j - 1].equals(U' ') && input[i - 1] == U' ') + cells[j - 1].append_space(); + } + if (i + 1 < input.size() && j + 1 < cells.size()) { + if (cells[j + 1].equals(U' ') && input[i + 1] != U' ') + cells[j + 1] = {0, false}; + else if (!cells[j + 1].equals(U' ') && input[i + 1] == U' ') + cells[j].append_space(); + } + ++j; + } + } + std::u32string out; + out.reserve(cells.size()); + for (auto cell : cells) { + if (cell.value) + out += cell.value; + if (cell.appended_space) + out += U' '; + } + return collapse(out); +} +std::string finish(const std::u32string &original, std::string_view output) { + return encode(restore(original, detokenize(decode(output)))); +} +} // namespace tn_text diff --git a/tools/text_processing_deployment/native_runtime/text_tables.h b/tools/text_processing_deployment/native_runtime/text_tables.h new file mode 100644 index 000000000..7c049d4be --- /dev/null +++ b/tools/text_processing_deployment/native_runtime/text_tables.h @@ -0,0 +1,473 @@ +// Generated by export_text_tables.py; do not edit. +// Python Unicode 15.0.0; sacremoses 0.2.0. +// Sacremoses-derived tables are covered by LICENSE.sacremoses. +#pragma once +namespace tn_text { +struct Range { char32_t first, last; }; +inline constexpr Range whitespace[] = { + {0x9,0xd}, + {0x1c,0x20}, + {0x85,0x85}, + {0xa0,0xa0}, + {0x1680,0x1680}, + {0x2000,0x200a}, + {0x2028,0x2029}, + {0x202f,0x202f}, + {0x205f,0x205f}, + {0x3000,0x3000}, +}; +inline constexpr Range alpha[] = { + {0x41,0x5a}, + {0x61,0x7a}, + {0xaa,0xaa}, + {0xb5,0xb5}, + {0xba,0xba}, + {0xc0,0xd6}, + {0xd8,0xf6}, + {0xf8,0x2c1}, + {0x2c6,0x2d1}, + {0x2e0,0x2e4}, + {0x2ec,0x2ec}, + {0x2ee,0x2ee}, + {0x345,0x345}, + {0x370,0x374}, + {0x376,0x377}, + {0x37a,0x37d}, + {0x37f,0x37f}, + {0x386,0x386}, + {0x388,0x38a}, + {0x38c,0x38c}, + {0x38e,0x3a1}, + {0x3a3,0x3f5}, + {0x3f7,0x481}, + {0x48a,0x52f}, + {0x531,0x556}, + {0x559,0x559}, + {0x561,0x587}, + {0x5b0,0x5bd}, + {0x5bf,0x5bf}, + {0x5c1,0x5c2}, + {0x5c4,0x5c5}, + {0x5c7,0x5c7}, + {0x5d0,0x5ea}, + {0x5f0,0x5f2}, + {0x610,0x61a}, + {0x620,0x657}, + {0x659,0x65f}, + {0x66e,0x6d3}, + {0x6d5,0x6dc}, + {0x6e1,0x6e8}, + {0x6ed,0x6ef}, + {0x6fa,0x6fc}, + {0x6ff,0x6ff}, + {0x710,0x73f}, + {0x74d,0x7b1}, + {0x7ca,0x7ea}, + {0x7f4,0x7f5}, + {0x7fa,0x7fa}, + {0x800,0x817}, + {0x81a,0x82c}, + {0x840,0x858}, + {0x8a0,0x8b2}, + {0x8e4,0x8e9}, + {0x8f0,0x93b}, + {0x93d,0x94c}, + {0x94e,0x950}, + {0x955,0x963}, + {0x971,0x983}, + {0x985,0x98c}, + {0x98f,0x990}, + {0x993,0x9a8}, + {0x9aa,0x9b0}, + {0x9b2,0x9b2}, + {0x9b6,0x9b9}, + {0x9bd,0x9c4}, + {0x9c7,0x9c8}, + {0x9cb,0x9cc}, + {0x9ce,0x9ce}, + {0x9d7,0x9d7}, + {0x9dc,0x9dd}, + {0x9df,0x9e3}, + {0x9f0,0x9f1}, + {0xa01,0xa03}, + {0xa05,0xa0a}, + {0xa0f,0xa10}, + {0xa13,0xa28}, + {0xa2a,0xa30}, + {0xa32,0xa33}, + {0xa35,0xa36}, + {0xa38,0xa39}, + {0xa3e,0xa42}, + {0xa47,0xa48}, + {0xa4b,0xa4c}, + {0xa51,0xa51}, + {0xa59,0xa5c}, + {0xa5e,0xa5e}, + {0xa70,0xa75}, + {0xa81,0xa83}, + {0xa85,0xa8d}, + {0xa8f,0xa91}, + {0xa93,0xaa8}, + {0xaaa,0xab0}, + {0xab2,0xab3}, + {0xab5,0xab9}, + {0xabd,0xac5}, + {0xac7,0xac9}, + {0xacb,0xacc}, + {0xad0,0xad0}, + {0xae0,0xae3}, + {0xb01,0xb03}, + {0xb05,0xb0c}, + {0xb0f,0xb10}, + {0xb13,0xb28}, + {0xb2a,0xb30}, + {0xb32,0xb33}, + {0xb35,0xb39}, + {0xb3d,0xb44}, + {0xb47,0xb48}, + {0xb4b,0xb4c}, + {0xb56,0xb57}, + {0xb5c,0xb5d}, + {0xb5f,0xb63}, + {0xb71,0xb71}, + {0xb82,0xb83}, + {0xb85,0xb8a}, + {0xb8e,0xb90}, + {0xb92,0xb95}, + {0xb99,0xb9a}, + {0xb9c,0xb9c}, + {0xb9e,0xb9f}, + {0xba3,0xba4}, + {0xba8,0xbaa}, + {0xbae,0xbb9}, + {0xbbe,0xbc2}, + {0xbc6,0xbc8}, + {0xbca,0xbcc}, + {0xbd0,0xbd0}, + {0xbd7,0xbd7}, + {0xc00,0xc03}, + {0xc05,0xc0c}, + {0xc0e,0xc10}, + {0xc12,0xc28}, + {0xc2a,0xc39}, + {0xc3d,0xc44}, + {0xc46,0xc48}, + {0xc4a,0xc4c}, + {0xc55,0xc56}, + {0xc58,0xc59}, + {0xc60,0xc63}, + {0xc81,0xc83}, + {0xc85,0xc8c}, + {0xc8e,0xc90}, + {0xc92,0xca8}, + {0xcaa,0xcb3}, + {0xcb5,0xcb9}, + {0xcbd,0xcc4}, + {0xcc6,0xcc8}, + {0xcca,0xccc}, + {0xcd5,0xcd6}, + {0xcde,0xcde}, + {0xce0,0xce3}, + {0xcf1,0xcf2}, + {0xd01,0xd03}, + {0xd05,0xd0c}, + {0xd0e,0xd10}, + {0xd12,0xd3a}, + {0xd3d,0xd44}, + {0xd46,0xd48}, + {0xd4a,0xd4c}, + {0xd4e,0xd4e}, + {0xd57,0xd57}, + {0xd60,0xd63}, + {0xd7a,0xd7f}, + {0xd82,0xd83}, + {0xd85,0xd96}, + {0xd9a,0xdb1}, + {0xdb3,0xdbb}, + {0xdbd,0xdbd}, + {0xdc0,0xdc6}, + {0xdcf,0xdd4}, + {0xdd6,0xdd6}, + {0xdd8,0xddf}, + {0xdf2,0xdf3}, + {0xe01,0xe3a}, + {0xe40,0xe46}, + {0xe4d,0xe4d}, + {0xe81,0xe82}, + {0xe84,0xe84}, + {0xe87,0xe88}, + {0xe8a,0xe8a}, + {0xe8d,0xe8d}, + {0xe94,0xe97}, + {0xe99,0xe9f}, + {0xea1,0xea3}, + {0xea5,0xea5}, + {0xea7,0xea7}, + {0xeaa,0xeab}, + {0xead,0xeb9}, + {0xebb,0xebd}, + {0xec0,0xec4}, + {0xec6,0xec6}, + {0xecd,0xecd}, + {0xedc,0xedf}, + {0xf00,0xf00}, + {0xf40,0xf47}, + {0xf49,0xf6c}, + {0xf71,0xf81}, + {0xf88,0xf97}, + {0xf99,0xfbc}, + {0x1000,0x1036}, + {0x1038,0x1038}, + {0x103b,0x103f}, + {0x1050,0x1062}, + {0x1065,0x1068}, + {0x106e,0x1086}, + {0x108e,0x108e}, + {0x109c,0x109d}, + {0x10a0,0x10c5}, + {0x10c7,0x10c7}, + {0x10cd,0x10cd}, + {0x10d0,0x10fa}, + {0x10fc,0x1248}, + {0x124a,0x124d}, + {0x1250,0x1256}, + {0x1258,0x1258}, + {0x125a,0x125d}, + {0x1260,0x1288}, + {0x128a,0x128d}, + {0x1290,0x12b0}, + {0x12b2,0x12b5}, + {0x12b8,0x12be}, + {0x12c0,0x12c0}, + {0x12c2,0x12c5}, + {0x12c8,0x12d6}, + {0x12d8,0x1310}, + {0x1312,0x1315}, + {0x1318,0x135a}, + {0x135f,0x135f}, + {0x1380,0x138f}, + {0x13a0,0x13f4}, + {0x1401,0x166c}, + {0x166f,0x167f}, + {0x1681,0x169a}, + {0x16a0,0x16ea}, + {0x16ee,0x16f8}, + {0x1700,0x170c}, + {0x170e,0x1713}, + {0x1720,0x1733}, + {0x1740,0x1753}, + {0x1760,0x176c}, + {0x176e,0x1770}, + {0x1772,0x1773}, + {0x1780,0x17b3}, + {0x17b6,0x17c8}, + {0x17d7,0x17d7}, + {0x17dc,0x17dc}, + {0x1820,0x1877}, + {0x1880,0x18aa}, + {0x18b0,0x18f5}, + {0x1900,0x191e}, + {0x1920,0x192b}, + {0x1930,0x1938}, + {0x1950,0x196d}, + {0x1970,0x1974}, + {0x1980,0x19ab}, + {0x19b0,0x19c9}, + {0x1a00,0x1a1b}, + {0x1a20,0x1a5e}, + {0x1a61,0x1a74}, + {0x1aa7,0x1aa7}, + {0x1b00,0x1b33}, + {0x1b35,0x1b43}, + {0x1b45,0x1b4b}, + {0x1b80,0x1ba9}, + {0x1bac,0x1baf}, + {0x1bba,0x1be5}, + {0x1be7,0x1bf1}, + {0x1c00,0x1c35}, + {0x1c4d,0x1c4f}, + {0x1c5a,0x1c7d}, + {0x1ce9,0x1cec}, + {0x1cee,0x1cf3}, + {0x1cf5,0x1cf6}, + {0x1d00,0x1dbf}, + {0x1de7,0x1df4}, + {0x1e00,0x1f15}, + {0x1f18,0x1f1d}, + {0x1f20,0x1f45}, + {0x1f48,0x1f4d}, + {0x1f50,0x1f57}, + {0x1f59,0x1f59}, + {0x1f5b,0x1f5b}, + {0x1f5d,0x1f5d}, + {0x1f5f,0x1f7d}, + {0x1f80,0x1fb4}, + {0x1fb6,0x1fbc}, + {0x1fbe,0x1fbe}, + {0x1fc2,0x1fc4}, + {0x1fc6,0x1fcc}, + {0x1fd0,0x1fd3}, + {0x1fd6,0x1fdb}, + {0x1fe0,0x1fec}, + {0x1ff2,0x1ff4}, + {0x1ff6,0x1ffc}, + {0x2071,0x2071}, + {0x207f,0x207f}, + {0x2090,0x209c}, + {0x2102,0x2102}, + {0x2107,0x2107}, + {0x210a,0x2113}, + {0x2115,0x2115}, + {0x2119,0x211d}, + {0x2124,0x2124}, + {0x2126,0x2126}, + {0x2128,0x2128}, + {0x212a,0x212d}, + {0x212f,0x2139}, + {0x213c,0x213f}, + {0x2145,0x2149}, + {0x214e,0x214e}, + {0x2160,0x2188}, + {0x24b6,0x24e9}, + {0x25cc,0x25cc}, + {0x2c00,0x2c2e}, + {0x2c30,0x2c5e}, + {0x2c60,0x2ce4}, + {0x2ceb,0x2cee}, + {0x2cf2,0x2cf3}, + {0x2d00,0x2d25}, + {0x2d27,0x2d27}, + {0x2d2d,0x2d2d}, + {0x2d30,0x2d67}, + {0x2d6f,0x2d6f}, + {0x2d80,0x2d96}, + {0x2da0,0x2da6}, + {0x2da8,0x2dae}, + {0x2db0,0x2db6}, + {0x2db8,0x2dbe}, + {0x2dc0,0x2dc6}, + {0x2dc8,0x2dce}, + {0x2dd0,0x2dd6}, + {0x2dd8,0x2dde}, + {0x2de0,0x2dff}, + {0x2e2f,0x2e2f}, + {0x3006,0x3006}, + {0x3031,0x3035}, + {0x303c,0x303c}, + {0x3041,0x3096}, + {0x309d,0x309f}, + {0x30a1,0x30fa}, + {0x30fc,0x30ff}, + {0x3105,0x312d}, + {0x3131,0x318e}, + {0x31a0,0x31ba}, + {0x31f0,0x31ff}, + {0xa000,0xa48c}, + {0xa4d0,0xa4fd}, + {0xa500,0xa60c}, + {0xa610,0xa61f}, + {0xa62a,0xa62b}, + {0xa640,0xa66e}, + {0xa674,0xa67b}, + {0xa67f,0xa69d}, + {0xa69f,0xa6ef}, + {0xa717,0xa71f}, + {0xa722,0xa788}, + {0xa78b,0xa78e}, + {0xa790,0xa7ad}, + {0xa7b0,0xa7b1}, + {0xa7f7,0xa801}, + {0xa803,0xa805}, + {0xa807,0xa80a}, + {0xa80c,0xa827}, + {0xa840,0xa873}, + {0xa880,0xa8c3}, + {0xa8f2,0xa8f7}, + {0xa8fb,0xa8fb}, + {0xa90a,0xa92a}, + {0xa930,0xa952}, + {0xa960,0xa97c}, + {0xa980,0xa9b2}, + {0xa9b4,0xa9bf}, + {0xa9cf,0xa9cf}, + {0xa9e0,0xa9e4}, + {0xa9e6,0xa9ef}, + {0xa9fa,0xa9fe}, + {0xaa00,0xaa36}, + {0xaa40,0xaa4d}, + {0xaa60,0xaa76}, + {0xaa7a,0xaa7a}, + {0xaa7e,0xaabe}, + {0xaac0,0xaac0}, + {0xaac2,0xaac2}, + {0xaadb,0xaadd}, + {0xaae0,0xaaef}, + {0xaaf2,0xaaf5}, + {0xab01,0xab06}, + {0xab09,0xab0e}, + {0xab11,0xab16}, + {0xab20,0xab26}, + {0xab28,0xab2e}, + {0xab30,0xab5a}, + {0xab5c,0xab5f}, + {0xab64,0xab65}, + {0xabc0,0xabea}, + {0xd7b0,0xd7c6}, + {0xd7cb,0xd7fb}, + {0xfb00,0xfb06}, + {0xfb13,0xfb17}, + {0xfb1d,0xfb28}, + {0xfb2a,0xfb36}, + {0xfb38,0xfb3c}, + {0xfb3e,0xfb3e}, + {0xfb40,0xfb41}, + {0xfb43,0xfb44}, + {0xfb46,0xfbb1}, + {0xfbd3,0xfd3d}, + {0xfd50,0xfd8f}, + {0xfd92,0xfdc7}, + {0xfdf0,0xfdfb}, + {0xfe70,0xfe74}, + {0xfe76,0xfefc}, + {0xff21,0xff3a}, + {0xff41,0xff5a}, + {0xff66,0xffbe}, + {0xffc2,0xffc7}, + {0xffca,0xffcf}, + {0xffd2,0xffd7}, + {0xffda,0xffdc}, +}; +inline constexpr Range currency_symbols[] = { + {0xa,0xa}, + {0x24,0x24}, + {0xa2,0xa5}, + {0x58f,0x58f}, + {0x60b,0x60b}, + {0x9f2,0x9f3}, + {0x9fb,0x9fb}, + {0xaf1,0xaf1}, + {0xbf9,0xbf9}, + {0xe3f,0xe3f}, + {0x17db,0x17db}, + {0x20a0,0x20bd}, + {0xa838,0xa838}, + {0xfdfc,0xfdfc}, + {0xfe69,0xfe69}, + {0xff04,0xff04}, + {0xffe0,0xffe1}, + {0xffe5,0xffe6}, +}; +inline constexpr Range cjk[] = { + {0x1100,0x11ff}, + {0x2e80,0xa4cf}, + {0xa840,0xa87f}, + {0xac00,0xd7af}, + {0xf900,0xfaff}, + {0xfe30,0xfe4f}, + {0xff65,0xffdc}, + {0x17000,0x18aff}, + {0x1b000,0x1b12f}, + {0x1b170,0x1b2ff}, + {0x20000,0x2ffff}, +}; +} From 88073468a050b39bc59edf5cf70ad3c92ba37918 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 13:05:10 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../native_runtime/benchmark.py | 11 ++-- .../native_runtime/byte_transducer.py | 15 ++--- .../native_runtime/test_byte_transducer.py | 60 ++++--------------- .../native_runtime/test_text_processing.py | 53 +++++----------- 4 files changed, 36 insertions(+), 103 deletions(-) diff --git a/tools/text_processing_deployment/native_runtime/benchmark.py b/tools/text_processing_deployment/native_runtime/benchmark.py index debe3cbea..4ee108ea2 100644 --- a/tools/text_processing_deployment/native_runtime/benchmark.py +++ b/tools/text_processing_deployment/native_runtime/benchmark.py @@ -39,17 +39,14 @@ def main() -> None: seconds["native_normalization"] += time.perf_counter() - begin if output is None: counts["no_path"] += 1 - differences.write( - json.dumps({"written": written, "error": "no path"}) - + "\n" - ) + differences.write(json.dumps({"written": written, "error": "no path"}) + "\n") continue reference = row["spoken"] if output != reference: counts["exact_differences"] += 1 - counts["word_token_differences"] += re.findall( - r"\w+", output - ) != re.findall(r"\w+", reference) + counts["word_token_differences"] += re.findall(r"\w+", output) != re.findall( + r"\w+", reference + ) differences.write( json.dumps( { diff --git a/tools/text_processing_deployment/native_runtime/byte_transducer.py b/tools/text_processing_deployment/native_runtime/byte_transducer.py index af8623022..df2be6b57 100644 --- a/tools/text_processing_deployment/native_runtime/byte_transducer.py +++ b/tools/text_processing_deployment/native_runtime/byte_transducer.py @@ -141,8 +141,7 @@ def __init__(self, library: Path, grammar_directory: Path) -> None: self.lib.tn_pipeline_normalize.argtypes = self.lib.tn_pipeline_rewrite.argtypes self.lib.tn_pipeline_normalize.restype = ctypes.c_int paths = [ - (grammar_directory / (name + ".bin")).absolute() - for name in ("tagger", "verbalizer", "postprocessor") + (grammar_directory / (name + ".bin")).absolute() for name in ("tagger", "verbalizer", "postprocessor") ] self.handle = self.lib.tn_pipeline_load(*(str(path).encode() for path in paths)) if not self.handle: @@ -172,9 +171,7 @@ def normalize_batch(self, texts: list[str]) -> list[str]: function.argtypes = self.lib.tn_pipeline_rewrite.argtypes function.restype = ctypes.c_int output, size = ctypes.c_void_p(), ctypes.c_size_t() - if function( - self.handle, frames, len(frames), ctypes.byref(output), ctypes.byref(size) - ): + if function(self.handle, frames, len(frames), ctypes.byref(output), ctypes.byref(size)): raise RuntimeError(self.lib.tn_error().decode()) data = ctypes.string_at(output.value, size.value) result, position = [], 0 @@ -197,12 +194,8 @@ def _normalize(self, text: str, *, complete: bool) -> str | None: encoded = text.encode() output = ctypes.c_void_p() size = ctypes.c_size_t() - function = ( - self.lib.tn_pipeline_normalize if complete else self.lib.tn_pipeline_rewrite - ) - status = function( - self.handle, encoded, len(encoded), ctypes.byref(output), ctypes.byref(size) - ) + function = self.lib.tn_pipeline_normalize if complete else self.lib.tn_pipeline_rewrite + status = function(self.handle, encoded, len(encoded), ctypes.byref(output), ctypes.byref(size)) if status < 0: raise RuntimeError(self.lib.tn_error().decode()) if status: diff --git a/tools/text_processing_deployment/native_runtime/test_byte_transducer.py b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py index 7d9c74734..442f2da94 100644 --- a/tools/text_processing_deployment/native_runtime/test_byte_transducer.py +++ b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py @@ -48,11 +48,7 @@ def check_graph(self, graph: pynini.Fst, inputs: list[str]) -> None: actual = runtime.rewrite(text) lattice = pynini.compose(pynini.accep(pynini.escape(text)), graph) expected_cost = ( - float( - pynini.shortestdistance(lattice, reverse=True)[ - lattice.start() - ] - ) + float(pynini.shortestdistance(lattice, reverse=True)[lattice.start()]) if lattice.start() >= 0 else float("inf") ) @@ -60,15 +56,9 @@ def check_graph(self, graph: pynini.Fst, inputs: list[str]) -> None: if actual is not None: output, cost = actual self.assertAlmostEqual(cost, expected_cost, places=5) - constrained = pynini.compose( - lattice, pynini.accep(pynini.escape(output)) - ) + constrained = pynini.compose(lattice, pynini.accep(pynini.escape(output))) self.assertGreaterEqual(constrained.start(), 0) - output_cost = float( - pynini.shortestdistance(constrained, reverse=True)[ - constrained.start() - ] - ) + output_cost = float(pynini.shortestdistance(constrained, reverse=True)[constrained.start()]) self.assertAlmostEqual(output_cost, expected_cost, places=5) finally: runtime.close() @@ -133,9 +123,7 @@ def test_native_pipeline_field_reordering(self) -> None: tags = 'tokens { money { currency: "dollars" integer: "twelve" } }' reordered = ' tokens { money { integer: "twelve" currency: "dollars" } } ' export_fst(pynini.cross("input", tags), self.root / "tagger.bin") - export_fst( - pynini.cross(reordered, "twelve dollars"), self.root / "verbalizer.bin" - ) + export_fst(pynini.cross(reordered, "twelve dollars"), self.root / "verbalizer.bin") export_fst( pynini.cross("twelve dollars", "TWELVE DOLLARS"), self.root / "postprocessor.bin", @@ -152,9 +140,7 @@ def test_native_pipeline_preserves_explicit_field_order(self) -> None: tags = 'tokens { money { currency: "dollars" integer: "twelve" preserve_order: true } }' forbidden = ' tokens { money { integer: "twelve" currency: "dollars" preserve_order: true } } ' export_fst(pynini.cross("input", tags), self.root / "tagger.bin") - export_fst( - pynini.cross(forbidden, "twelve dollars"), self.root / "verbalizer.bin" - ) + export_fst(pynini.cross(forbidden, "twelve dollars"), self.root / "verbalizer.bin") export_fst(pynini.accep("twelve dollars"), self.root / "postprocessor.bin") native = NativePipeline(self.library, self.root) try: @@ -184,9 +170,7 @@ def test_float_tie_after_shared_merge(self) -> None: (2, 1, 0, 0, 0.0), (1, 3, 0, 0, 1e8), ]: - graph.add_arc( - src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst) - ) + graph.add_arc(src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst)) path = self.root / "merge.bin" export_fst(graph, path) runtime = ByteTransducer(self.library, path) @@ -210,9 +194,7 @@ def test_prepared_lookahead_arc_order(self) -> None: (2, 4, 97, 122, 0.0), (3, 4, 0, 113, 0.0), ]: - graph.add_arc( - src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst) - ) + graph.add_arc(src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst)) path = self.root / "lookahead.bin" export_fst(graph, path) for mode, expected in [("transducer", "yq"), ("lookahead", "xz")]: @@ -255,9 +237,7 @@ def test_projected_output_against_top_rewrite(self) -> None: expected = None actual = runtime.rewrite(text) with self.subTest(text=text): - self.assertEqual( - None if actual is None else actual[0], expected - ) + self.assertEqual(None if actual is None else actual[0], expected) finally: runtime.close() @@ -275,9 +255,7 @@ def test_lookahead_phase_merge_before_float_rounding(self) -> None: (3, 1, 98, 121, -10), (4, 2, 0, 120, 0), ]: - graph.add_arc( - src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst) - ) + graph.add_arc(src, pynini.Arc(ilabel, olabel, pynini.Weight("tropical", weight), dst)) path = self.root / "phase-merge.bin" export_fst(graph, path) runtime = ByteTransducer(self.library, path, mode="lookahead") @@ -328,11 +306,7 @@ def test_random_exact_outputs(self) -> None: rng.choice([0, 120, 121, 122]), pynini.Weight( "tropical", - 0 - if case >= 600 - else rng.choice( - [0, 0, 0, -1e-8, 0.1, 0.2, 1e8, -10] - ), + 0 if case >= 600 else rng.choice([0, 0, 0, -1e-8, 0.1, 0.2, 1e8, -10]), ), ids[later], ), @@ -361,16 +335,10 @@ def test_random_exact_outputs(self) -> None: expected = None else: lattice = pynini.accep(text) @ graph - expected = ( - pynini.shortestpath(lattice).string() - if lattice.start() >= 0 - else None - ) + expected = pynini.shortestpath(lattice).string() if lattice.start() >= 0 else None actual = runtime.rewrite(text) with self.subTest(case=case, mode=mode, text=text): - self.assertEqual( - None if actual is None else actual[0], expected - ) + self.assertEqual(None if actual is None else actual[0], expected) finally: runtime.close() @@ -388,9 +356,7 @@ def test_unweighted_reference_queue_order(self) -> None: (2, 0, 0, 120), (2, 1, 0, 121), ]: - graph.add_arc( - src, pynini.Arc(ilabel, olabel, pynini.Weight.one("tropical"), dst) - ) + graph.add_arc(src, pynini.Arc(ilabel, olabel, pynini.Weight.one("tropical"), dst)) graph.arcsort("ilabel") expected = pynini.shortestpath(pynini.accep("b") @ graph).string() self.assertEqual(expected, "yx") diff --git a/tools/text_processing_deployment/native_runtime/test_text_processing.py b/tools/text_processing_deployment/native_runtime/test_text_processing.py index c0b6cd318..6612ad214 100644 --- a/tools/text_processing_deployment/native_runtime/test_text_processing.py +++ b/tools/text_processing_deployment/native_runtime/test_text_processing.py @@ -12,13 +12,12 @@ from pathlib import Path import pynini -from .byte_transducer import NativePipeline, export_fst -from nemo_text_processing.text_normalization.data_loader_utils import ( - post_process_punct, - pre_process, -) from sacremoses import MosesDetokenizer +from nemo_text_processing.text_normalization.data_loader_utils import post_process_punct, pre_process + +from .byte_transducer import NativePipeline, export_fst + class TextProcessingTest(unittest.TestCase): @classmethod @@ -84,9 +83,7 @@ def stage(self, stage: int, written: str, output: str) -> str: results = [] for lib in self.libs: result, length = ctypes.c_void_p(), ctypes.c_size_t() - status = lib.tn_text_stage( - stage, a, len(a), b, len(b), ctypes.byref(result), ctypes.byref(length) - ) + status = lib.tn_text_stage(stage, a, len(a), b, len(b), ctypes.byref(result), ctypes.byref(length)) self.assertEqual(status, 0) results.append(ctypes.string_at(result.value, length.value).decode()) self.assertEqual(results[0], results[1], "SIMD/scalar mismatch") @@ -97,9 +94,7 @@ def check_pair(self, written: str, output: str) -> None: self.assertEqual(self.stage(0, written, ""), pre_process(written).strip()) detokenized = self.detokenizer.detokenize([output], unescape=False) self.assertEqual(self.stage(1, "", output), detokenized) - expected = " ".join( - post_process_punct(input=written, normalized_text=detokenized).split() - ) + expected = " ".join(post_process_punct(input=written, normalized_text=detokenized).split()) self.assertEqual(self.stage(2, written, output), expected) def test_public_punctuation_and_unicode_cases(self) -> None: @@ -198,9 +193,7 @@ def test_invalid_utf8_and_nul(self) -> None: self.assertEqual(status, -1) def test_stream_and_complete_api(self) -> None: - export_fst( - pynini.cross("x", 'tokens { name: "word" }'), self.root / "tagger.bin" - ) + export_fst(pynini.cross("x", 'tokens { name: "word" }'), self.root / "tagger.bin") export_fst( pynini.cross(' tokens { name: "word" } ', "word"), self.root / "verbalizer.bin", @@ -209,12 +202,8 @@ def test_stream_and_complete_api(self) -> None: values = ["x", " \nx\t", "", "\u2003x\u00a0"] native = NativePipeline(self.library, self.root) try: - self.assertEqual( - [native.normalize(s) for s in values], ["word", "word", "", "word"] - ) - self.assertEqual( - native.normalize_batch(values), ["word", "word", "", "word"] - ) + self.assertEqual([native.normalize(s) for s in values], ["word", "word", "", "word"]) + self.assertEqual(native.normalize_batch(values), ["word", "word", "", "word"]) self.assertEqual(native.normalize_batch([]), []) with self.assertRaisesRegex(RuntimeError, "Batch record 1"): native.normalize_batch(["x", "\0", "x"]) @@ -231,23 +220,15 @@ def test_stream_and_complete_api(self) -> None: self.assertEqual(status, -1) self.assertIsNone(output.value) self.assertEqual(size.value, 0) - self.assertEqual( - native.normalize_batch(values), ["word", "word", "", "word"] - ) + self.assertEqual(native.normalize_batch(values), ["word", "word", "", "word"]) finally: native.close() source = self.root / "input.frames" target = self.root / "output.frames" - source.write_bytes( - b"".join(struct.pack(" None: normalizer.verbalizer = SimpleNamespace(fst=pynini.cross("unreachable", "word")) normalizer.post_processor = SimpleNamespace(fst=pynini.accep("word")) for tags in ['tokens { name: "word" }', 'tokens { name: "" }']: - normalizer.tagger = SimpleNamespace( - fst=pynini.cross(pynini.escape("[ x ]"), tags) - ) + normalizer.tagger = SimpleNamespace(fst=pynini.cross(pynini.escape("[ x ]"), tags)) for name, component in [ ("tagger", normalizer.tagger), ("verbalizer", normalizer.verbalizer), @@ -283,9 +262,7 @@ def test_failed_verbalization_matches_reference(self) -> None: export_fst(component.fst, self.root / (name + ".bin")) native = NativePipeline(self.library, self.root) try: - expected = normalizer.normalize( - "[x]", punct_pre_process=True, punct_post_process=True - ) + expected = normalizer.normalize("[x]", punct_pre_process=True, punct_post_process=True) self.assertEqual(native.normalize("[x]"), expected) finally: native.close() From d52b81cec5c6de84456b97db19fed93bec618b41 Mon Sep 17 00:00:00 2001 From: Albert Zeyer Date: Sat, 17 Oct 2026 10:41:35 -0700 Subject: [PATCH 3/6] Document malformed corpus input and test native batch recovery Signed-off-by: Albert Zeyer --- .../native_runtime/README.md | 3 ++ .../native_runtime/byte_transducer.py | 8 ++++- .../native_runtime/test_text_processing.py | 36 +++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) diff --git a/tools/text_processing_deployment/native_runtime/README.md b/tools/text_processing_deployment/native_runtime/README.md index 071890b65..024536fcf 100644 --- a/tools/text_processing_deployment/native_runtime/README.md +++ b/tools/text_processing_deployment/native_runtime/README.md @@ -51,6 +51,9 @@ NativePipeline loads tagger.bin, verbalizer.bin and postprocessor.bin from a sup Its normalize method performs complete English processing; normalize_core exposes the transducer stages for diagnostics. normalize_batch uses one native call for a list of records, with no Python callbacks during normalization. Batches use the same framing as the standalone driver, allow at most 64 MiB of input or output, and return no partial result on failure. +Embedded NUL characters can occur in crawl text and are rejected even inside an otherwise readable sentence. +Corpus preparation should reject and count those records before batching; the runtime does not strip characters or silently discard records. +A record failure identifies the zero-based batch index, publishes no partial output and leaves the handle usable for the next call. ByteTransducer evaluates individual exported graphs and returns both output and path cost. The C ABI also supports native callers without Python. diff --git a/tools/text_processing_deployment/native_runtime/byte_transducer.py b/tools/text_processing_deployment/native_runtime/byte_transducer.py index df2be6b57..6f6d96989 100644 --- a/tools/text_processing_deployment/native_runtime/byte_transducer.py +++ b/tools/text_processing_deployment/native_runtime/byte_transducer.py @@ -162,7 +162,13 @@ def normalize(self, text: str) -> str | None: return self._normalize(text, complete=True) def normalize_batch(self, texts: list[str]) -> list[str]: - """Run a bounded batch in native code and fail without returning partial results.""" + """Run a bounded batch in native code and fail without returning partial results. + + Embedded NUL characters are unsupported, including inside otherwise valid text. + A failing record raises RuntimeError with its zero-based batch index. + The handle remains usable after failure; no partial result is returned. + Corpus callers should explicitly reject and count malformed records before batching. + """ if not self.handle: raise RuntimeError("Pipeline is closed") encoded = [text.encode() for text in texts] diff --git a/tools/text_processing_deployment/native_runtime/test_text_processing.py b/tools/text_processing_deployment/native_runtime/test_text_processing.py index 6612ad214..0695bff62 100644 --- a/tools/text_processing_deployment/native_runtime/test_text_processing.py +++ b/tools/text_processing_deployment/native_runtime/test_text_processing.py @@ -239,6 +239,42 @@ def test_stream_and_complete_api(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn(b"Truncated input frame header", result.stderr) + def test_embedded_nul_batch_failure_and_reuse(self) -> None: + export_fst(pynini.cross("x", 'tokens { name: "word" }'), self.root / "tagger.bin") + export_fst(pynini.cross(' tokens { name: "word" } ', "word"), self.root / "verbalizer.bin") + export_fst(pynini.accep("word"), self.root / "postprocessor.bin") + # Exercise both sides of SIMD boundaries and NUL after multibyte UTF-8. + malformed = ["a" * n + "\0readable suffix" for n in (0, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65)] + malformed += ["René paid \0 twelve dollars.", "x\0", "\u2003\0\u00a0"] + for library in (self.library, self.scalar): + native = NativePipeline(library, self.root) + try: + for value in malformed: + with self.subTest(library=library.name, value=value): + with self.assertRaisesRegex(RuntimeError, "Embedded NUL is unsupported"): + native.normalize(value) + self.assertEqual(native.normalize("x"), "word") + for record in range(3): + values = ["x", "", "x"] + values[record] = value + with self.assertRaisesRegex( + RuntimeError, f"^Batch record {record}: Embedded NUL is unsupported$" + ): + native.normalize_batch(values) + frames = b"".join( + struct.pack(" None: from types import SimpleNamespace From 6ef88d867a6abc00748a6c0b729a8f03cfede9c5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 17:42:16 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../native_runtime/test_text_processing.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tools/text_processing_deployment/native_runtime/test_text_processing.py b/tools/text_processing_deployment/native_runtime/test_text_processing.py index 0695bff62..5dd90aa63 100644 --- a/tools/text_processing_deployment/native_runtime/test_text_processing.py +++ b/tools/text_processing_deployment/native_runtime/test_text_processing.py @@ -261,9 +261,7 @@ def test_embedded_nul_batch_failure_and_reuse(self) -> None: RuntimeError, f"^Batch record {record}: Embedded NUL is unsupported$" ): native.normalize_batch(values) - frames = b"".join( - struct.pack(" Date: Sat, 17 Oct 2026 11:12:08 -0700 Subject: [PATCH 5/6] Match Pynini NUL semantics in native English normalization Signed-off-by: Albert Zeyer --- .../native_runtime/README.md | 8 +- .../native_runtime/byte_transducer.cc | 26 +++++- .../native_runtime/byte_transducer.py | 4 +- .../native_runtime/test_byte_transducer.py | 44 +++++++++- .../native_runtime/test_text_processing.py | 80 ++++++++++--------- .../native_runtime/text_processing.h | 2 - 6 files changed, 114 insertions(+), 50 deletions(-) diff --git a/tools/text_processing_deployment/native_runtime/README.md b/tools/text_processing_deployment/native_runtime/README.md index 024536fcf..8b9c4f773 100644 --- a/tools/text_processing_deployment/native_runtime/README.md +++ b/tools/text_processing_deployment/native_runtime/README.md @@ -51,8 +51,10 @@ NativePipeline loads tagger.bin, verbalizer.bin and postprocessor.bin from a sup Its normalize method performs complete English processing; normalize_core exposes the transducer stages for diagnostics. normalize_batch uses one native call for a list of records, with no Python callbacks during normalization. Batches use the same framing as the standalone driver, allow at most 64 MiB of input or output, and return no partial result on failure. -Embedded NUL characters can occur in crawl text and are rejected even inside an otherwise readable sentence. -Corpus preparation should reject and count those records before batching; the runtime does not strip characters or silently discard records. +Embedded NUL characters follow Pynini's epsilon interpretation at transducer input. +They remain available in the original text for punctuation handling and failed-verbalization fallback; containing NUL does not discard a record. +The evaluator retains the original input's emptiness and trailing-epsilon phase when choosing tied paths. +The low-level direct-transducer mode still rejects NUL because its composition-filter tie ordering is not implemented; the complete English pipeline uses prepared lookahead. A record failure identifies the zero-based batch index, publishes no partial output and leaves the handle usable for the next call. ByteTransducer evaluates individual exported graphs and returns both output and path cost. The C ABI also supports native callers without Python. @@ -84,7 +86,7 @@ The audited dependency set is NeMo text processing 1.2.0, Pynini 2.1.6.post1, Py Changing grammar exports or dependencies requires fresh differential validation. TNBYTE02 exports retain cached acyclicity rather than recomputing it, since that cache affects reference queue choice. Re-export legacy TNBYTE01 grammars before direct-mode unweighted execution; missing queue metadata is an error there. -Malformed UTF-8, NUL, unsupported labels and resource-limit failures are explicit errors, not silent fallbacks. +Malformed UTF-8, unsupported labels and resource-limit failures are explicit errors, not silent fallbacks. Relevant implementation references: diff --git a/tools/text_processing_deployment/native_runtime/byte_transducer.cc b/tools/text_processing_deployment/native_runtime/byte_transducer.cc index 78219e319..da97da2ed 100644 --- a/tools/text_processing_deployment/native_runtime/byte_transducer.cc +++ b/tools/text_processing_deployment/native_runtime/byte_transducer.cc @@ -514,6 +514,8 @@ class Program { DagWorkspace dag; bool push_prefix = false; + // Removed input epsilon arcs still affect AutoQueue and terminal lookahead phase. + bool input_had_nul = false, trailing_nul = false; // Specialize record execution so each arc does not retest the same mode flags. template bool rewrite_dag_impl(const char *data, size_t size, float &best_cost) { @@ -563,7 +565,7 @@ class Program { if (!can_consume(terminal, nextlabel)) return; } - bool prefix = lookahead && nextpos < size && last_epsilon; + bool prefix = lookahead && (nextpos < size || trailing_nul) && last_epsilon; uint32_t target = dag.get(terminal * 2 + prefix, nextpos); if (edges.size() >= 10000000) throw std::runtime_error("DAG edge limit"); @@ -644,7 +646,7 @@ class Program { } // AutoQueue tests known acyclicity before falling back to unweighted LIFO. // Prepared lookahead retains the acyclic property for empty input only. - bool lifo = unweighted && !(lookahead ? size == 0 : known_acyclic); + bool lifo = unweighted && !(lookahead ? size == 0 && !input_had_nul : known_acyclic); if (lifo && !lookahead && !has_queue_properties) throw std::runtime_error( "Re-export legacy grammar with cached queue properties"); @@ -727,6 +729,20 @@ class Program { return rewrite_dag_impl(data, size, cost); } bool rewrite(const char *data, size_t size, float &cost) { + // Pynini compiles byte zero as epsilon, not a consuming input label. + // Keep it in the original text for punctuation and failed-verbalization fallback. + input_had_nul = std::memchr(data, 0, size) != nullptr; + trailing_nul = size && !data[size - 1]; + if (input_had_nul) { + if (!push_prefix && !project_output) + throw std::runtime_error("NUL requires lookahead or output-projected mode"); + std::string input; + input.reserve(size); + for (size_t i = 0; i < size; ++i) + if (data[i]) + input.push_back(data[i]); + return rewrite_dag(input.data(), input.size(), cost); + } return rewrite_dag(data, size, cost); } @@ -986,7 +1002,9 @@ class Pipeline { if (!tagger.rewrite(text, length, cost)) return false; auto tokens = TagParser(tagger.result).parse(); - for (size_t begin = 0; begin < tokens.size();) { + // NeMo attempts verbalization even when the tag parser returns no tokens. + size_t begin = 0; + do { size_t end = begin, count = 1; while (end < tokens.size()) { size_t n = permutation_count({tokens[end]}); @@ -1021,7 +1039,7 @@ class Pipeline { return false; } begin = end; - } + } while (begin < tokens.size()); result = collapse_spaces(result); if (!result.empty()) { if (!postprocessor.rewrite(result.data(), result.size(), cost)) diff --git a/tools/text_processing_deployment/native_runtime/byte_transducer.py b/tools/text_processing_deployment/native_runtime/byte_transducer.py index 6f6d96989..680b43482 100644 --- a/tools/text_processing_deployment/native_runtime/byte_transducer.py +++ b/tools/text_processing_deployment/native_runtime/byte_transducer.py @@ -164,10 +164,10 @@ def normalize(self, text: str) -> str | None: def normalize_batch(self, texts: list[str]) -> list[str]: """Run a bounded batch in native code and fail without returning partial results. - Embedded NUL characters are unsupported, including inside otherwise valid text. + Embedded NUL follows Pynini's epsilon interpretation at transducer input. + Original text is retained for punctuation handling and failed-verbalization fallback. A failing record raises RuntimeError with its zero-based batch index. The handle remains usable after failure; no partial result is returned. - Corpus callers should explicitly reject and count malformed records before batching. """ if not self.handle: raise RuntimeError("Pipeline is closed") diff --git a/tools/text_processing_deployment/native_runtime/test_byte_transducer.py b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py index 442f2da94..0427db4b5 100644 --- a/tools/text_processing_deployment/native_runtime/test_byte_transducer.py +++ b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py @@ -327,7 +327,11 @@ def test_random_exact_outputs(self) -> None: for mode in ["transducer", "output"]: runtime = ByteTransducer(self.library, path, mode=mode) try: - for text in ["", "a", "b", "ab", "ba", "aa", "bbb", "abab"]: + for text in ["", "a", "b", "ab", "ba", "aa", "bbb", "abab", "\0", "a\0", "a\0b", "\0b"]: + if mode == "transducer" and "\0" in text: + with self.assertRaisesRegex(RuntimeError, "NUL requires"): + runtime.rewrite(text) + continue if mode == "output": try: expected = rewrite.top_rewrite(text, graph) @@ -342,6 +346,44 @@ def test_random_exact_outputs(self) -> None: finally: runtime.close() + def test_nul_preserves_lookahead_queue_and_terminal_phase(self) -> None: + # Expected outputs were checked against OpenFst's prepared lookahead matcher. + cases = [ + (10, {0: 2, 4: 0, 9: 0}, "a\0", "zxyzy", [ + (0, 0, 122, 1), (0, 98, 120, 9), (1, 0, 120, 2), (2, 0, 0, 3), + (2, 0, 0, 8), (2, 97, 0, 9), (3, 98, 0, 4), (3, 0, 0, 5), + (3, 0, 121, 7), (4, 98, 120, 5), (4, 0, 0, 7), (4, 0, 121, 9), + (5, 0, 120, 6), (5, 97, 121, 7), (5, 0, 122, 8), (6, 98, 0, 7), + (7, 0, 122, 8), (8, 0, 121, 9), + ]), + (12, {4: 0, 6: 2, 7: 0, 11: 0}, "\0", "yy", [ + (0, 98, 0, 1), (0, 0, 121, 2), (0, 98, 120, 11), (1, 0, 0, 2), + (2, 0, 120, 3), (2, 0, 121, 10), (3, 0, 0, 4), (4, 98, 121, 5), + (4, 97, 121, 9), (4, 0, 122, 11), (5, 97, 120, 6), (5, 98, 122, 9), + (6, 97, 122, 7), (7, 97, 122, 8), (7, 98, 122, 10), (7, 97, 0, 11), + (8, 0, 0, 9), (9, 98, 122, 10), (9, 0, 121, 11), (10, 0, 0, 11), + ]), + ] + for count, finals, text, expected, arcs in cases: + graph = pynini.Fst() + for _ in range(count): + graph.add_state() + graph.set_start(0) + for state, weight in finals.items(): + graph.set_final(state, pynini.Weight("tropical", weight)) + for source, ilabel, olabel, target in arcs: + graph.add_arc(source, pynini.Arc(ilabel, olabel, pynini.Weight.one("tropical"), target)) + graph.arcsort("ilabel") + path = self.root / "nul-queue.bin" + export_fst(graph, path) + runtime = ByteTransducer(self.library, path, mode="lookahead") + try: + self.assertEqual(runtime.rewrite(text)[0], expected) + runtime.rewrite(text.replace("\0", "")) + self.assertEqual(runtime.rewrite(text)[0], expected) + finally: + runtime.close() + def test_unweighted_reference_queue_order(self) -> None: # OpenFst AutoQueue chooses LIFO when acyclicity is not already known. graph = pynini.Fst() diff --git a/tools/text_processing_deployment/native_runtime/test_text_processing.py b/tools/text_processing_deployment/native_runtime/test_text_processing.py index 5dd90aa63..8ecec2550 100644 --- a/tools/text_processing_deployment/native_runtime/test_text_processing.py +++ b/tools/text_processing_deployment/native_runtime/test_text_processing.py @@ -169,15 +169,13 @@ def test_seeded_text_stage_equivalence(self) -> None: with self.subTest(written=a, output=b): self.check_pair(a, b) - def test_invalid_utf8_and_nul(self) -> None: + def test_invalid_utf8(self) -> None: for value in ( b"\xc0\x80", b"\xed\xa0\x80", b"\xf4\x90\x80\x80", b"\xff", b"\xe2\x82", - b"\0", - b"a" * 16 + b"\0", ): for lib in self.libs: result, length = ctypes.c_void_p(), ctypes.c_size_t() @@ -206,7 +204,7 @@ def test_stream_and_complete_api(self) -> None: self.assertEqual(native.normalize_batch(values), ["word", "word", "", "word"]) self.assertEqual(native.normalize_batch([]), []) with self.assertRaisesRegex(RuntimeError, "Batch record 1"): - native.normalize_batch(["x", "\0", "x"]) + native.normalize_batch(["x", "unmatched", "x"]) self.assertEqual(native.normalize_batch(["x"]), ["word"]) for malformed in [b"\x02\x00", struct.pack(" None: self.assertNotEqual(result.returncode, 0) self.assertIn(b"Truncated input frame header", result.stderr) - def test_embedded_nul_batch_failure_and_reuse(self) -> None: - export_fst(pynini.cross("x", 'tokens { name: "word" }'), self.root / "tagger.bin") - export_fst(pynini.cross(' tokens { name: "word" } ', "word"), self.root / "verbalizer.bin") - export_fst(pynini.accep("word"), self.root / "postprocessor.bin") - # Exercise both sides of SIMD boundaries and NUL after multibyte UTF-8. - malformed = ["a" * n + "\0readable suffix" for n in (0, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65)] - malformed += ["René paid \0 twelve dollars.", "x\0", "\u2003\0\u00a0"] - for library in (self.library, self.scalar): - native = NativePipeline(library, self.root) - try: - for value in malformed: - with self.subTest(library=library.name, value=value): - with self.assertRaisesRegex(RuntimeError, "Embedded NUL is unsupported"): - native.normalize(value) - self.assertEqual(native.normalize("x"), "word") - for record in range(3): - values = ["x", "", "x"] - values[record] = value - with self.assertRaisesRegex( - RuntimeError, f"^Batch record {record}: Embedded NUL is unsupported$" - ): - native.normalize_batch(values) - frames = b"".join(struct.pack(" None: + from types import SimpleNamespace + + from nemo_text_processing.text_normalization.normalize import Normalizer + from nemo_text_processing.text_normalization.token_parser import TokenParser + + values = ["a" * n + "\0word" for n in (0, 1, 15, 16, 17, 31, 32, 33, 63, 64, 65)] + values += ["René\0 word", "x\0", "\u2003\0\u00a0", "\0", " \0 \0 ", "[\0x]", "x \0 y"] + normalizer = Normalizer.__new__(Normalizer) + normalizer.lang = "en" + normalizer.parser = TokenParser() + normalizer.max_number_of_permutations_per_split = 729 + normalizer.moses_detokenizer = self.detokenizer + prepared = {pre_process(" ".join(value.split())).strip().replace("\0", "") for value in values} + normalizer.tagger = SimpleNamespace( + fst=pynini.union(*(pynini.cross(pynini.escape(value), 'tokens { name: "word" }') for value in prepared)) + ) + normalizer.verbalizer = SimpleNamespace(fst=pynini.cross(' tokens { name: "word" } ', "word")) + normalizer.post_processor = SimpleNamespace(fst=pynini.accep("word")) + # Also check the early-return path, which must retain the original NUL. + for failed_verbalization in (False, True): + if failed_verbalization: + normalizer.verbalizer.fst = pynini.cross("unreachable", "word") + for name in ("tagger", "verbalizer", "post_processor"): + filename = "postprocessor" if name == "post_processor" else name + export_fst(getattr(normalizer, name).fst, self.root / (filename + ".bin")) + expected = [ + normalizer.normalize(" ".join(value.split()), punct_pre_process=True, punct_post_process=True) + for value in values + ] + for library in (self.library, self.scalar): + native = NativePipeline(library, self.root) + try: + with self.subTest(library=library.name, failed_verbalization=failed_verbalization): + self.assertEqual([native.normalize(value) for value in values], expected) + self.assertEqual(native.normalize_batch(values), expected) + self.assertEqual(native.normalize_batch(values), expected) + finally: + native.close() def test_failed_verbalization_matches_reference(self) -> None: from types import SimpleNamespace @@ -286,7 +290,7 @@ def test_failed_verbalization_matches_reference(self) -> None: normalizer.moses_detokenizer = self.detokenizer normalizer.verbalizer = SimpleNamespace(fst=pynini.cross("unreachable", "word")) normalizer.post_processor = SimpleNamespace(fst=pynini.accep("word")) - for tags in ['tokens { name: "word" }', 'tokens { name: "" }']: + for tags in [" ", 'tokens { name: "word" }', 'tokens { name: "" }']: normalizer.tagger = SimpleNamespace(fst=pynini.cross(pynini.escape("[ x ]"), tags)) for name, component in [ ("tagger", normalizer.tagger), diff --git a/tools/text_processing_deployment/native_runtime/text_processing.h b/tools/text_processing_deployment/native_runtime/text_processing.h index d4deb4703..1833e65f0 100644 --- a/tools/text_processing_deployment/native_runtime/text_processing.h +++ b/tools/text_processing_deployment/native_runtime/text_processing.h @@ -102,8 +102,6 @@ std::u32string decode(std::string_view s) { throw std::runtime_error("Invalid UTF-8 code point"); out.push_back(c); } - if (out.find(U'\0') != std::u32string::npos) - throw std::runtime_error("Embedded NUL is unsupported"); return out; } std::string encode(const std::u32string &s) { From d5c045c75f003571cb70330b9653bca17b241fec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:12:46 +0000 Subject: [PATCH 6/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../native_runtime/test_byte_transducer.py | 68 +++++++++++++++---- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/tools/text_processing_deployment/native_runtime/test_byte_transducer.py b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py index 0427db4b5..241d389b2 100644 --- a/tools/text_processing_deployment/native_runtime/test_byte_transducer.py +++ b/tools/text_processing_deployment/native_runtime/test_byte_transducer.py @@ -349,20 +349,60 @@ def test_random_exact_outputs(self) -> None: def test_nul_preserves_lookahead_queue_and_terminal_phase(self) -> None: # Expected outputs were checked against OpenFst's prepared lookahead matcher. cases = [ - (10, {0: 2, 4: 0, 9: 0}, "a\0", "zxyzy", [ - (0, 0, 122, 1), (0, 98, 120, 9), (1, 0, 120, 2), (2, 0, 0, 3), - (2, 0, 0, 8), (2, 97, 0, 9), (3, 98, 0, 4), (3, 0, 0, 5), - (3, 0, 121, 7), (4, 98, 120, 5), (4, 0, 0, 7), (4, 0, 121, 9), - (5, 0, 120, 6), (5, 97, 121, 7), (5, 0, 122, 8), (6, 98, 0, 7), - (7, 0, 122, 8), (8, 0, 121, 9), - ]), - (12, {4: 0, 6: 2, 7: 0, 11: 0}, "\0", "yy", [ - (0, 98, 0, 1), (0, 0, 121, 2), (0, 98, 120, 11), (1, 0, 0, 2), - (2, 0, 120, 3), (2, 0, 121, 10), (3, 0, 0, 4), (4, 98, 121, 5), - (4, 97, 121, 9), (4, 0, 122, 11), (5, 97, 120, 6), (5, 98, 122, 9), - (6, 97, 122, 7), (7, 97, 122, 8), (7, 98, 122, 10), (7, 97, 0, 11), - (8, 0, 0, 9), (9, 98, 122, 10), (9, 0, 121, 11), (10, 0, 0, 11), - ]), + ( + 10, + {0: 2, 4: 0, 9: 0}, + "a\0", + "zxyzy", + [ + (0, 0, 122, 1), + (0, 98, 120, 9), + (1, 0, 120, 2), + (2, 0, 0, 3), + (2, 0, 0, 8), + (2, 97, 0, 9), + (3, 98, 0, 4), + (3, 0, 0, 5), + (3, 0, 121, 7), + (4, 98, 120, 5), + (4, 0, 0, 7), + (4, 0, 121, 9), + (5, 0, 120, 6), + (5, 97, 121, 7), + (5, 0, 122, 8), + (6, 98, 0, 7), + (7, 0, 122, 8), + (8, 0, 121, 9), + ], + ), + ( + 12, + {4: 0, 6: 2, 7: 0, 11: 0}, + "\0", + "yy", + [ + (0, 98, 0, 1), + (0, 0, 121, 2), + (0, 98, 120, 11), + (1, 0, 0, 2), + (2, 0, 120, 3), + (2, 0, 121, 10), + (3, 0, 0, 4), + (4, 98, 121, 5), + (4, 97, 121, 9), + (4, 0, 122, 11), + (5, 97, 120, 6), + (5, 98, 122, 9), + (6, 97, 122, 7), + (7, 97, 122, 8), + (7, 98, 122, 10), + (7, 97, 0, 11), + (8, 0, 0, 9), + (9, 98, 122, 10), + (9, 0, 121, 11), + (10, 0, 0, 11), + ], + ), ] for count, finals, text, expected, arcs in cases: graph = pynini.Fst()