Add knap, a byte level BPE tokenizer in pure Mojo

recipes/knap/recipe.yaml

@@ -0,0 +1,155 @@
+# SPDX-License-Identifier: EUPL-1.2
+# Copyright 2026 Olaf Yunus Laitinen Imanov
+#
+# Conda recipe for Knap, a byte level BPE tokenizer written in pure Mojo.
+#
+#     https://github.com/olaflaitinen/knap
+#
+# The artefact is a precompiled Mojo package plus a command line tool. The
+# compiler's own documentation is careful to say a precompiled package is
+# tied to the exact compiler version that produced it, which is why the
+# compiler is pinned exactly in the run requirements rather than given a
+# range. A consumer on a different toolchain gets a solver error, which is
+# far better than a link error deep in their build. That is a deliberate
+# deviation from pin_compatible and the reasoning is here rather than in a
+# pull request comment, so it survives.
+#
+# This recipe is maintained in the Knap repository at
+# conda.recipe/recipe.yaml and copied here. scripts/check_recipe.py there
+# checks, on every push, that the declared version agrees with CITATION.cff,
+# that the compiler pin agrees with pixi.toml and pyproject.toml, that
+# source.rev is a commit in that repository, and that every path the build
+# script reads exists at that commit.
+#
+# Build it:
+#
+#     rattler-build build \
+#       --recipe recipes/knap/recipe.yaml \
+#       -c conda-forge \
+#       -c https://conda.modular.com/max \
+#       -c https://repo.prefix.dev/modular-community
+
+schema_version: 1
+
+context:
+  version: "1.0.0"
+  mojo_version: "1.0.0"
+
+package:
+  name: knap
+  version: ${{ version }}
+
+# A git source at a full commit SHA rather than a local path. Two reasons,
+# and the second is the one that matters. A path source cannot be built by
+# anyone but the author, so the modular-community repository, which holds
+# only the recipe, could not build it at all. And a SHA is reproducible in a
+# way a branch or a tag is not: a tag can be moved, a branch always is.
+#
+# The SHA below is the v1.0.0 release commit of the Knap repository. It is
+# updated as part of preparing a release, and Knap's own recipe gate
+# refuses a value that is not a commit there.
+source:
+  - git: https://github.com/olaflaitinen/knap.git
+    rev: e83f0ea34fb6d586551b7f9df226006d78dc1436
+
+build:
+  number: 0
+  # Noarch is wrong here even though a precompiled Mojo package contains no
+  # elaborated code. The package is tied to a compiler version rather than to
+  # an architecture, and conda has no way to express that, so it is built per
+  # platform and pinned to the compiler instead.
+  script:
+    interpreter: bash
+    content: |
+      set -euo pipefail
+
+      # Into $PREFIX/lib/mojo, which is the path that makes the package
+      # discoverable by the compiler without a -I flag.
+      mkdir -p "${PREFIX}/lib/mojo"
+      mojo precompile src/knap -o "${PREFIX}/lib/mojo/knap.mojoc"
+
+      # The command line tool, so that installing the package gives a person
+      # a working `knap` command rather than a library they have to write a
+      # program against. Counting tokens is the most common thing anyone
+      # does with a tokenizer, and it should not require a toolchain.
+      mkdir -p "${PREFIX}/bin"
+      mojo build -I src -I cli -o "${PREFIX}/bin/knap" cli/main.mojo
+
+      # Shell completions, in the directories each shell searches. A tool
+      # that installs a binary and no completions is a tool people type the
+      # long way for years.
+      mkdir -p "${PREFIX}/share/bash-completion/completions"
+      cp cli/completions/knap.bash \
+        "${PREFIX}/share/bash-completion/completions/knap"
+      mkdir -p "${PREFIX}/share/zsh/site-functions"
+      cp cli/completions/_knap "${PREFIX}/share/zsh/site-functions/_knap"
+      mkdir -p "${PREFIX}/share/fish/vendor_completions.d"
+      cp cli/completions/knap.fish \
+        "${PREFIX}/share/fish/vendor_completions.d/knap.fish"
+
+requirements:
+  build:
+    - mojo-compiler ==${{ mojo_version }}
+  host:
+    - mojo-compiler ==${{ mojo_version }}
+  # pin_compatible would be the idiomatic choice here and it is deliberately
+  # not used. It derives a range from whatever version resolved at build
+  # time, and a range is exactly what a precompiled Mojo artefact cannot
+  # honour: the compiler's own documentation says a .mojoc file may not be
+  # compatible with another compiler version, and the ABI is explicitly not
+  # stable. An exact pin turns that into a solver error at install time.
+  run:
+    - mojo-compiler ==${{ mojo_version }}
+
+tests:
+  - script:
+      interpreter: bash
+      content: |
+        set -euo pipefail
+        test -f "${PREFIX}/lib/mojo/knap.mojoc"
+        test -x "${PREFIX}/bin/knap"
+        test -f "${PREFIX}/share/bash-completion/completions/knap"
+        test -f "${PREFIX}/share/zsh/site-functions/_knap"
+        test -f "${PREFIX}/share/fish/vendor_completions.d/knap.fish"
+        # The version the binary reports must be the version the package
+        # claims. Printing it proves only that the binary runs; a package
+        # built from the wrong revision would pass that and fail this.
+        test "$("${PREFIX}/bin/knap" version)" = "${{ version }}"
+
+        # No -I here, deliberately. The build installs into
+        # ${PREFIX}/lib/mojo precisely so the compiler finds the package
+        # without one, and a test that passes the flag would still pass if
+        # that promise were broken. This line is what makes the claim above
+        # the build script an assertion rather than a comment.
+        mojo run smoke.mojo
+    files:
+      recipe:
+        - smoke.mojo
+
+about:
+  homepage: https://github.com/olaflaitinen/knap
+  repository: https://github.com/olaflaitinen/knap.git
+  documentation: https://github.com/olaflaitinen/knap/blob/main/README.md
+  license: EUPL-1.2
+  license_file: LICENSE
+  summary: A byte level BPE tokenizer written in pure Mojo, with no Python interpreter at run time.
+  description: |
+    Knap is a byte level Byte Pair Encoding tokenizer written in pure Mojo,
+    verified to produce byte identical output to tiktoken. All seven
+    tiktoken encodings are supported and each was checked against tiktoken
+    itself: 191762320 tokens over a 110 MB corpus, every token id in every
+    encoding, and tens of millions of fuzzed inputs.
+
+    Installing it gives both a Mojo library and a `knap` command for
+    counting, encoding, and decoding from a shell. Vocabularies are not
+    bundled and are fetched separately; `knap help` says where the tool
+    looks for them.
+
+    Correctness is the product. Knap is faster than tiktoken on three of the
+    four distinct encode behaviours on the published machine and slower than
+    rs-bpe everywhere rs-bpe runs, and its benchmarks publish the cases
+    where the alternatives win.
+
+extra:
+  maintainers:
+    - olaflaitinen

recipes/knap/smoke.mojo

@@ -0,0 +1,65 @@
+# =============================================================================
+# Project     : Knap, a pure Mojo byte level BPE tokenizer
+# File        : conda.recipe/smoke.mojo
+# Purpose     : Package acceptance test. Imports Knap from the installed
+#               conda package and decodes one token, with no source tree.
+# Stage       : Distribution. See docs/PACKAGING.md
+# Depends on  : knap.flat_vocab, resolved from $PREFIX/lib/mojo
+# Invariants  : Imports nothing that needs a vocabulary file, so the test
+#               passes or fails on the package rather than on a download.
+# -----------------------------------------------------------------------------
+# Author      : Olaf Yunus Laitinen Imanov <yunus.imanov@metropolia.fi>
+# ORCID       : 0009-0006-5184-0810
+# Affiliation : School of Information and Communication Technology,
+#               Metropolia University of Applied Sciences
+# -----------------------------------------------------------------------------
+# SPDX-License-Identifier: EUPL-1.2
+# Copyright 2026 Olaf Yunus Laitinen Imanov
+# =============================================================================
+"""Package acceptance test for the Knap conda package.
+
+A package that builds but cannot be imported without the source tree is not
+a package, and nothing in the build itself would notice. This program is run
+by the recipe's test section against the installed artefact only, with
+`-I $PREFIX/lib/mojo` and no path into the repository.
+
+It is deliberately small. It builds a two byte vocabulary by hand and
+decodes one token, which exercises the import path, the precompiled package,
+and one real code path, and needs no vocabulary download to do it. A larger
+test here would be testing Knap, which the suite already does, rather than
+testing the package.
+
+    mojo run -I "$PREFIX/lib/mojo" smoke.mojo
+"""
+
+from knap.flat_vocab import FlatVocab
+
+
+def main() raises:
+    """Decode one token from a hand built vocabulary.
+
+    Raises:
+        Error: if the package cannot be imported, or if the decode returns
+            anything other than the two bytes that went in.
+    """
+    var data = List[UInt8]()
+    data.append(UInt8(72))  # H
+    data.append(UInt8(105))  # i
+
+    var offsets: List[Int] = [0]
+    var lengths: List[Int] = [2]
+    var vocabulary = FlatVocab(data^, offsets^, lengths^)
+
+    var decoded = vocabulary.decode([0])
+    if decoded != String("Hi"):
+        raise Error(
+            String(
+                t"knap package smoke test: decoded '{decoded}', expected 'Hi'"
+            )
+        )
+    print("knap package smoke test: import and decode both work")
+
+
+# =============================================================================
+# End of file: conda.recipe/smoke.mojo
+# =============================================================================